I'm trying to write a test http-server for a personal IoT project I"ve got going. My project has its own web-server that responds to certain 'page load' requests. Now I'm trying to expand the web-page itself. Now I could develop the actual server itself simultaneously but downloading it to the IoT device every iteration is extraordinarily slow so I've decided to write a test-server on my machine instead.
The test-server is written in python. The code below is the test-server in its entirety:
import http.server
import socketserver
import string
PORT = 8000
class MyHandler(http.server.SimpleHTTPRequestHandler):
def __init__(self):
self.pageFuncs = { "/Lights": self.doLightsPage,
"/Show": self.doShowPage,
"/Static": self.doShowTypePage,
"/Effects": self.doShowTypePage,
"/Playlist": self.doShowTypePage,
"/Select": self.doControlTypePage,
"/Controls": self.doControlTypePage,
"/query": self.doQueryResultsPage }
self.STATIC_SHOW_TYPE = 0
self.EFFECTS_SHOW_TYPE = 1
self.PLAYLIST_SHOW_TYPE = 2
self.SELECT_CTRL_TYPE = 0
self.CONTROL_CTRL_TYPE = 1
self.LightsOn = 0
self.LightShowOn = 0
self.showType = STATIC_SHOW_TYPE
self.ctrlType = SELECT_CTRL_TYPE
def do_GET(self):
try:
page_name = self.path
if (page_name in pageFuncs):
print (page_name)
self.send_response(200)
self.send_header("Content-type", "text/json")
self.end_headers()
self.pageFuncs[page_name](self)
return
else:
super().do_GET()
except IOError:
self.send_error(404,'File Not Found: %s' % self.path)
except :
pass
def doLightsPage(self):
self.bLightsOn ^= 1
return doQueryResultsPage()
def doShowPage(self):
self.bLightShowOn ^= 1
return doQueryResultsPage()
def doShowTypePage(self):
if (self.path == '/Static') :
self.showType = STATIC_SHOW_TYPE
elif (self.path == '/Effects') :
self.showType = EFFECTS_SHOW_TYPE
else :
self.showType = PLAYLIST_SHOW_TYPE
return doQueryResultsPage()
def doControlTypePage(self):
if (self.path == '/Select') :
self.ctrlType = SELECT_CTRL_TYPE
else :
self.ctrlType = CONTROL_CTRL_TYPE
return doQueryResultsPage()
def doQueryResultsPage(self):
json_data = '{ \"power\": {}, \"show\": {},'
json_data += '\"show_type\": {}, \"control_type\": {} }'
json_data = format(json_data, bLightsOn, bLightShowOn, showType, ctrlType);
self.write(json_data)
def main():
try:
server = socketserver.TCPServer(("", PORT), MyHandler)
print ('started httpserver...')
server.serve_forever()
except KeyboardInterrupt:
print ('^C received, shutting down server')
server.socket.close()
if __name__ == '__main__':
main()
The point of the test is to receive a valid request; change a value and return a json string back reflecting the internal state of the server.
However, as soon as I try to load any of the pages, I get the following error message:
Exception happened during processing of request from ('127.0.0.1', 50149)
Traceback (most recent call last):
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.752.0_x64__qbz5n2kfra8p0\lib\socketserver.py", line 316, in _handle_request_noblock
self.process_request(request, client_address)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.752.0_x64__qbz5n2kfra8p0\lib\socketserver.py", line 347, in process_request
self.finish_request(request, client_address)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.752.0_x64__qbz5n2kfra8p0\lib\socketserver.py", line 360, in finish_request
self.RequestHandlerClass(request, client_address, self)
TypeError: __init__() takes 0 positional arguments but 4 were given
So my question is, what am I doing wrong?
I've taken the code format from the following stackoverflow topic which appears to be trying to do the same things, maybe I'm misunderstanding something?
I am trying to make a makeblock mblock implementation with python and i found this api on github but it uses python 2.7 and i am using python 3.8 so i am trying to configure the code so it can run on python 3.8 but i ran into an error which is
init mBot
bot
<lib.mBot.mSerial object at 0x02CE9898>
start with serial
<lib.mBot.mHID object at 0x02E2B310>
self.device = mHID()
'mHID' object has no attribute 'dict'
Exception in thread Thread-2:
self.start()
Traceback (most recent call last):
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 154, in __onRead
start with HID
--------------------
n = self.device.inWaiting()
<lib.mBot.mBot object at 0x002FF640>
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 96, in inWaiting
--------------------
Error in sys.excepthook:
buf = self.dict.device.read(64)
Traceback (most recent call last):
AttributeError: 'mHID' object has no attribute 'dict'
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 134, in excepthook
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\abdsak11\AppData\Local\Programs\Python\Python38-32\lib\threading.py", line 932, in _bootstrap_inner
self.close()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 142, in close
self.device.close()
self.run()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 106, in close
File "C:\Users\abdsak11\AppData\Local\Programs\Python\Python38-32\lib\threading.py", line 870, in run
self.dict.device.close()
AttributeError: 'mHID' object has no attribute 'dict'
Original exception was:
Traceback (most recent call last):
File "c:/Users/abdsak11/OneDrive - Lärande/Dokument/GitHub/python-for-mbot/test.py", line 15, in <module>
self._target(*self._args, **self._kwargs)
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 163, in __onRead
bot.doMove(50,50)
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 179, in doMove
self.close()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 142, in close
self.__writePackage(bytearray([0xff,0x55,0x7,0x0,0x2,0x5]+self.short2bytes(-leftSpeed)+self.short2bytes(rightSpeed)))
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 278, in short2bytes
self.device.close()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 106, in close
return [ord(val[0]),ord(val[1])]
TypeError: ord() expected string of length 1, but int found
self.dict.device.close()
AttributeError: 'mHID' object has no attribute 'dict'
'mHID' object has no attribute 'dict'
Exception in thread Thread-1:
Traceback (most recent call last):
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 154, in __onRead
n = self.device.inWaiting()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 96, in inWaiting
buf = self.dict.device.read(64)
AttributeError: 'mHID' object has no attribute 'dict'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\abdsak11\AppData\Local\Programs\Python\Python38-32\lib\threading.py", line 932, in _bootstrap_inner
self.run()
File "C:\Users\abdsak11\AppData\Local\Programs\Python\Python38-32\lib\threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 163, in __onRead
self.close()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 142, in close
self.device.close()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 106, in close
self.dict.device.close()
AttributeError: 'mHID' object has no attribute 'dict'
So what i am trying to do is to control and upload code to my mbot via usb COM3 serial and thats the error that i am getting i tried everthing but its not working and i cant seem to figure out the cause of the error.
mbot.py
this is the main api that i found
import serial
import sys,time
import signal
from time import ctime,sleep
import glob,struct
from multiprocessing import Process,Manager,Array
import threading
import hid
class mSerial():
ser = None
def __init__(self):
print (self)
def start(self, port):
self.ser = serial.Serial(port,115200)
def device(self):
return self.ser
def serialPorts(self):
if sys.platform.startswith('win'):
ports = ['COM%s' % (i + 1) for i in range(256)]
elif sys.platform.startswith('linux') or sys.platform.startswith('cygwin'):
ports = glob.glob('/dev/tty[A-Za-z]*')
elif sys.platform.startswith('darwin'):
ports = glob.glob('/dev/tty.*')
else:
raise EnvironmentError('Unsupported platform')
result = []
for port in ports:
s = serial.Serial()
s.port = port
s.close()
result.append(port)
return result
def writePackage(self,package):
self.ser.write(package)
sleep(0.01)
def read(self):
return self.ser.read()
def isOpen(self):
return self.ser.isOpen()
def inWaiting(self):
return self.ser.inWaiting()
def close(self):
self.ser.close()
class mHID():
def __init__(self):
print (self)
def start(self):
print ("starting start \n\n")
self.manager = Manager()
print("manager pass \n\n")
self.dict = self.manager.dict()
print("dict pass \n\n")
self.dict.device = hid.device()
print("dict device pass \n\n")
self.dict.device.open(0x0416, 0xffff)
print("dict device open pass \n\n")
self.dict.device.hid_set_nonblocking(self.device,1)
print ("start")
self.buffer = []
self.bufferIndex = 0
def enumerate(self):
print ("enumerate")
for dev in self.dict.device.enumerate():
print ('------------------------------------------------------------')
print (dev.description())
def writePackage(self,package):
buf = []
buf += [0, len(package)]
for i in range(len(package)):
buf += [package[i]]
n = self.dict.device.write(buf)
sleep(0.01)
def read(self):
c = self.buffer[0]
self.buffer = self.buffer[1:]
return chr(c)
def isOpen(self):
return True
def inWaiting(self):
buf = self.dict.device.read(64)
l = 0
if len(buf)>0:
l = buf[0]
if l>0:
for i in range(0,l):
self.buffer += [buf[i+1]]
return len(self.buffer)
def close(self):
self.dict.device.close()
class mBot():
def __init__(self):
print ("init mBot")
signal.signal(signal.SIGINT, self.exit)
self.manager = Manager()
self.__selectors = self.manager.dict()
self.buffer = []
self.bufferIndex = 0
self.isParseStart = False
self.exiting = False
self.isParseStartIndex = 0
def startWithSerial(self, port):
self.device = mSerial()
self.device.start(port)
self.start()
def startWithHID(self):
self.device = mHID()
print ("self.device = mHID()\n \n")
self.device.start()
print ("self.device.start()\n \n")
self.start()
print ("self.start()\n \n")
def excepthook(self, exctype, value, traceback):
self.close()
def start(self):
sys.excepthook = self.excepthook
th = threading.Thread(target=self.__onRead,args=(self.onParse,))
th.start()
def close(self):
self.device.close()
def exit(self, signal, frame):
self.exiting = True
sys.exit(0)
def __onRead(self,callback):
while 1:
if(self.exiting==True):
break
try:
if self.device.isOpen()==True:
n = self.device.inWaiting()
for i in range(n):
r = ord(self.device.read())
callback(r)
sleep(0.01)
else:
sleep(0.5)
except Exception as ex:
print (str(ex))
self.close()
sleep(1)
def __writePackage(self,pack):
self.device.writePackage(pack)
def doRGBLed(self,port,slot,index,red,green,blue):
self.__writePackage(bytearray([0xff,0x55,0x9,0x0,0x2,0x8,port,slot,index,red,green,blue]))
def doRGBLedOnBoard(self,index,red,green,blue):
self.doRGBLed(0x7,0x2,index,red,green,blue)
def doMotor(self,port,speed):
self.__writePackage(bytearray([0xff,0x55,0x6,0x0,0x2,0xa,port]+self.short2bytes(speed)))
def doMove(self,leftSpeed,rightSpeed):
self.__writePackage(bytearray([0xff,0x55,0x7,0x0,0x2,0x5]+self.short2bytes(-leftSpeed)+self.short2bytes(rightSpeed)))
def doServo(self,port,slot,angle):
self.__writePackage(bytearray([0xff,0x55,0x6,0x0,0x2,0xb,port,slot,angle]))
def doBuzzer(self,buzzer,time=0):
self.__writePackage(bytearray([0xff,0x55,0x7,0x0,0x2,0x22]+self.short2bytes(buzzer)+self.short2bytes(time)))
def doSevSegDisplay(self,port,display):
self.__writePackage(bytearray([0xff,0x55,0x8,0x0,0x2,0x9,port]+self.float2bytes(display)))
def doIROnBoard(self,message):
self.__writePackage(bytearray([0xff,0x55,len(message)+3,0x0,0x2,0xd,message]))
def requestLightOnBoard(self,extID,callback):
self.requestLight(extID,8,callback)
def requestLight(self,extID,port,callback):
self.__doCallback(extID,callback)
self.__writePackage(bytearray([0xff,0x55,0x4,extID,0x1,0x3,port]))
def requestButtonOnBoard(self,extID,callback):
self.__doCallback(extID,callback)
self.__writePackage(bytearray([0xff,0x55,0x4,extID,0x1,0x1f,0x7]))
def requestIROnBoard(self,extID,callback):
self.__doCallback(extID,callback)
self.__writePackage(bytearray([0xff,0x55,0x3,extID,0x1,0xd]))
def requestUltrasonicSensor(self,extID,port,callback):
self.__doCallback(extID,callback)
self.__writePackage(bytearray([0xff,0x55,0x4,extID,0x1,0x1,port]))
def requestLineFollower(self,extID,port,callback):
self.__doCallback(extID,callback)
self.__writePackage(bytearray([0xff,0x55,0x4,extID,0x1,0x11,port]))
def onParse(self, byte):
position = 0
value = 0
self.buffer+=[byte]
bufferLength = len(self.buffer)
if bufferLength >= 2:
if (self.buffer[bufferLength-1]==0x55 and self.buffer[bufferLength-2]==0xff):
self.isParseStart = True
self.isParseStartIndex = bufferLength-2
if (self.buffer[bufferLength-1]==0xa and self.buffer[bufferLength-2]==0xd and self.isParseStart==True):
self.isParseStart = False
position = self.isParseStartIndex+2
extID = self.buffer[position]
position+=1
type = self.buffer[position]
position+=1
# 1 byte 2 float 3 short 4 len+string 5 double
if type == 1:
value = self.buffer[position]
if type == 2:
value = self.readFloat(position)
if(value<-255 or value>1023):
value = 0
if type == 3:
value = self.readShort(position)
if type == 4:
value = self.readString(position)
if type == 5:
value = self.readDouble(position)
if(type<=5):
self.responseValue(extID,value)
self.buffer = []
def readFloat(self, position):
v = [self.buffer[position], self.buffer[position+1],self.buffer[position+2],self.buffer[position+3]]
return struct.unpack('<f', struct.pack('4B', *v))[0]
def readShort(self, position):
v = [self.buffer[position], self.buffer[position+1]]
return struct.unpack('<h', struct.pack('2B', *v))[0]
def readString(self, position):
l = self.buffer[position]
position+=1
s = ""
for i in Range(l):
s += self.buffer[position+i].charAt(0)
return s
def readDouble(self, position):
v = [self.buffer[position], self.buffer[position+1],self.buffer[position+2],self.buffer[position+3]]
return struct.unpack('<f', struct.pack('4B', *v))[0]
def responseValue(self, extID, value):
self.__selectors["callback_"+str(extID)](value)
def __doCallback(self, extID, callback):
self.__selectors["callback_"+str(extID)] = callback
def float2bytes(self,fval):
val = struct.pack("f",fval)
return [ord(val[0]),ord(val[1]),ord(val[2]),ord(val[3])]
def short2bytes(self,sval):
val = struct.pack("h",sval)
return [ord(val[0]),ord(val[1])]
test.py
this is the test file that i am working with
from lib.mBot import *
if __name__ == "__main__":
bot = mBot()
print ("bot \n\n")
bot.startWithSerial("COM3")
print ("start with serial\n\n")
bot.startWithHID()
print ("start with HID \n\n")
print(f'--------------------\n')
print (bot)
print(f'--------------------\n')
bot.doMove(50,50)
I believe your trouble is here:
class mHID():
def __init__(self):
print (self)
def start(self):
# ...
self.dict = self.manager.dict()
# ...
def close(self):
self.dict.device.close()
You should always define your instance attributes in __init__(), at least with an initial value of None, even if you plan to change them later.
Additionally, whenever you have a situation where "A must be called before B", you should write the code to properly handle when B is called without A. In this case, what should close() do if start() was never called? For simplicity, I'm just having it do nothing, but you may consider raising an exception.
class mHID():
def __init__(self):
print (self)
self.dct = None
def start(self):
# ...
self.dct = self.manager.dict()
# ...
def close(self):
if self.dct:
self.dct.device.close()
You'll notice I also renamed self.dict to self.dct. Although the former name may technically be allowable because of the self. namespace, dict is actually a built-in class. Under various circumstances, you could wind up dealing with shadowing. In the very least, it's confusing to always have to remind oneself that "this isn't the built-in dict, but an instance variable instead.
(Possible aside: is there a particular reason why you don't want to call start() from the initializer?)
WARNING: This is not a simple case of "copy and paste the answer". I've illustrated a couple of design principles here that you should incorporate into the rest of your code.
I am trying to populate lists self.images and self.servers with functions self.refresh_images_list and self.refresh_server_list using multiprocessing. I am doing this so when the object is created they will kick off async. I am using shared lists so the child copy will update the original objects list.
However, I am getting a Pickle error so I am pretty stuck.
class Account():
def __init__(self, username, api, pipe_object):
manager = Manager()
self.images = manager.list()
self.servers = manager.list()
self.images_timestamp = None
self.servers_timestamp = None
#needed a dictionary instead of
#list/tuple. This works best for
#the generator.
self.regions = {
"DFW" : pyrax.connect_to_cloudservers("DFW"),
"ORD" : pyrax.connect_to_cloudservers("ORD"),
"SYD" : pyrax.connect_to_cloudservers("SYD")
}
p1 = Process(target = self.refresh_server_list, args=())
p2 = Process(target = self.refresh_image_list, args=())
p1.start()
p2.start()
p1.join()
p2.join()
flavors = None
#multiprocessing shares lists only for __init__
#after __init__, we want to break the share
unshare_lists = False
def refresh_server_list(self):
if self.unshare_lists:
self.servers = []
self.servers_timestamp = time.strftime(
"%I:%M:%S", time.localtime()
)
with Redirect(self.pipe_object):
print "\nRefreshing server cache...hold on!"
for region, value in self.regions.iteritems():
region_servers = value.servers.list()
for region_servers in generator(region_servers, region):
self.servers.append(region_servers)
with Redirect(self.pipe_object):
print "\nServer cache completed!"
def server_list(self):
if not self.servers:
self.refresh_server_list()
with Redirect(self.pipe_object):
print_header("Cached Server List", "-")
for srv in self.servers:
print "\nName: %s" % srv.name
print "Created: %s" % srv.created
print "Progress: %s" % srv.progress
print "Status: %s" % srv.status
print "URL: %s" % srv.links[0]["href"]
print "Networks: %s" % srv.networks
print "\nLast Refresh time: %s" % self.servers_timestamp
def refresh_image_list(self):
if self.unshare_lists:
self.images = []
self.images_timestamp = time.strftime(
"%I:%M:%S", time.localtime()
)
with Redirect(self.pipe_object):
# print_header("Active Image List", "-")
print "\nRefreshing image cache...hold on!"
for region, value in self.regions.iteritems():
region_images = value.images.list()
for region_images in generator(region_images, region):
self.images.append(region_images)
with Redirect(self.pipe_object):
print "\nImage cache completed!"
def image_list(self):
if not self.images:
self.refresh_image_list()
with Redirect(self.pipe_object):
print_header("List Images", "-")
for img in self.images:
print (
str(self.images.index(img)+1) + ") "
+ "Name: %s\n ID: %s Status: %s" %
(img.name, img.id, img.status)
)
print "\nLast Refresh time: %s" % self.images_timestamp
The error I get:
Refreshing server cache...hold on!
Traceback (most recent call last):
File "menu.py", line 162, in <module>
main()
File "menu.py", line 156, in main
menus[value](hash_table, accounts)
File "menu.py", line 104, in menu
choices[value]()
File "/home/work/modules/classes.py", line 87, in server_list
self.refresh_server_list()
File "/home/work/modules/classes.py", line 80, in refresh_server_list
self.servers.append(region_servers)
File "<string>", line 2, in append
File "/usr/lib64/python2.7/multiprocessing/managers.py", line 758, in _callmethod
conn.send((self._id, methodname, args, kwds))
cPickle.PicklingError: Can't pickle <type 'instancemethod'>: attribute lookup __builtin__.instancemethod failed
I am getting the following error after a few hours of successful running.
Traceback (most recent call last):
File "/usr/lib/python2.6/threading.py", line 484, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.6/dist-packages/twisted/python/threadpool.py", line 210, in _worker
result = context.call(ctx, function, *args, **kwargs)
File "/usr/lib/python2.6/dist-packages/twisted/python/context.py", line 59, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/lib/python2.6/dist-packages/twisted/python/context.py", line 37, in callWithContext
return func(*args,**kw)
--- <exception caught here> ---
File "/usr/lib/python2.6/dist-packages/twisted/enterprise/adbapi.py", line 436, in _runInteraction
conn.rollback()
File "/usr/lib/python2.6/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback
self._connection.rollback()
_mysql_exceptions.OperationalError: (2006, 'MySQL server has gone away')
My code is something like this...
from twisted.internet import reactor, defer,threads
from twisted.enterprise import adbapi
dbpool = adbapi.ConnectionPool("MySQLdb", '192.168.1.102','test', 'test', 'test')
class Scanner:
def _execQuery(self,txn):
sql="SELECT tool_id,tool_name FROM tool_master"
txn.execute(sql)
result = txn.fetchall()
return result
def objCursor(self):
return dbpool.runInteraction(self._execQuery)
def printResult(self,result):
print "resssssssssssssssssss",result
reactor.callLater(3,self.deferExecute)
def deferExecute(self):
self.objCursor().addCallback(self.printResult)
Scanner()
class MyApp(object):
reactor.callInThread(Scanner().deferExecute)
reactor.run()
MyApp()
Can anyone tell me why I am getting this error?
can anyone tell me why I am getting this error.. because you're doing it wrong.
runInteraction runs the supplied function with an argument of a cursor to a transaction which is run in a thread. You shouldn't be calling reactor.callInThread(Scanner().deferExecute).
It's better to use a twisted.internet.task.LoopingCall, it will make sure that the call completes before the next is fired.
You're just running a query in your example, so you could just use ConnectionPool.runQuery instead of ConnectionPool.runInteraction.
Use errorBack functions to report on Exceptions.
Attempting to correct for your badly formatted code, I think you've got this:
from twisted.internet import reactor, defer,threads
from twisted.enterprise import adbapi
dbpool = adbapi.ConnectionPool("MySQLdb", '192.168.1.102','test', 'test', 'test')
class Scanner:
def _execQuery(self,txn):
sql="SELECT tool_id,tool_name FROM tool_master"
txn.execute(sql)
result = txn.fetchall()
return result
def objCursor(self):
return dbpool.runInteraction(self._execQuery)
def printResult(self,result):
print "resssssssssssssssssss",result
reactor.callLater(3,self.deferExecute)
def deferExecute(self):
self.objCursor().addCallback(self.printResult)
Scanner()
class MyApp(object):
reactor.callInThread(Scanner().deferExecute)
reactor.run()
MyApp()
When you probably need something like the following instead. If you're planning on writing a twisted Application will be easy to modify this Scanner class to inherit from twisted.application.service.Service.
from twisted.internet import reactor, defer, task
from twisted.enterprise import adbapi
class Scanner(object):
def __init__(self,dbpool=None):
self.dbpool = dbpool
self.loopCall = task.LoopingCall(self.myQuery)
def start(self):
print "Started scanner"
self.loopCall.start(3)
def stop(self):
print "Stopping scanner"
self.loopCall.stop()
def myQuery(self):
def interact(txn):
sql="SELECT tool_id,tool_name FROM tool_master"
txn.execute(sql)
return txn.fetchall()
d = self.dbpool.runInteraction(interact)
d.addCallbacks(self.printResult,self.printError)
def printResult(self,result):
print "Got Result: %r" % result
def printError(self,error):
print "Got Error: %r" % error
error.printTraceback()
if __name__ == '__main__':
from twisted.internet import reactor
dbpool = adbapi.ConnectionPool("MySQLdb", '192.168.1.102','test', 'test', 'test')
s = Scanner(dbpool)
reactor.callWhenRunning(s.start)
reactor.addSystemEventTrigger('before','shutdown',s.stop)
reactor.run()
After all the suggestion & help by Matt I have following code which is running successfully:
#!usr/bin/python
# Using the "dbmodule" from the previous example, create a ConnectionPool
from twisted.internet import reactor
from twisted.enterprise import adbapi
from twisted.internet import reactor, defer,threads
from twisted.python.threadpool import ThreadPool
import itertools
from twisted.internet.threads import deferToThread
from twisted.internet import reactor, defer, task
from tools.printTime import *
from tools.getVersion import *
from sh_log import *
concurrent = 30
finished=itertools.count(1)
reactor.suggestThreadPoolSize(concurrent)
#Creating Global Instance variables
path="tools"
lo=Log()
class ToolsBuilder:
def build(self,txn,tool,asset_id):
if tool:
print "\n"
try:
sql="select tool_filename from tool_master where tool_id = %s" %(tool,)
sql_asset="select asset_url from asset_master where asset_id = %s" %(asset_id,)
txn.execute(sql_asset)
asset_url = txn.fetchall()
log_date=lo.log_date()
txn.execute(sql)
result = txn.fetchall()
log='\n'+log_date+"::"+str(result[0][0])+ " tool object is created......\n"
lo.wfile(log)
temp=(path +'/' + str(result[0][0]))
if result:
if temp:
f=open(temp).read()
obj_tool=compile(f, 'a_filename', 'exec')
return obj_tool
except:
lo.wfile("Error in creating executable tool object......")
tb=ToolsBuilder()
class ToolsVectorGenerator:
def generate(self,txn,res_set={}):
v1=[]
for asset_id in res_set.iterkeys():
try:
obj_tools=[]
if asset_id:
print "asset_id..............................",asset_id
log_date=lo.log_date()
log=log_date+"::"+" \nVector generation for the asset number...:"+str(asset_id)
lo.wfile(log)
vector=[]
tools_arr=[]
obj_tools=[]
for tool in res_set[asset_id]:
if tool:
print "tool..............",tool
temp_tool=tb.build(txn,tool,asset_id)
print "temp_tool..........",temp_tool
#fetch data of tool setting.....
sql_tool_setting="select * from tool_asset_settings where tool_id =%s" %(tool,)
txn.execute(sql_tool_setting)
result_tool_setting = txn.fetchall()
tool_id=result_tool_setting[0][1]
t_id=int(tool_id)
tool_id_arr=[]
tool_id_arr.append(t_id)
tool_id_arr.append(result_tool_setting)
tool_id_arr.append(temp_tool)
tools_arr.append(tool_id_arr)
#fetch data from asset master
sql_asset="select asset_name from asset_master where asset_id=%s" %(asset_id,)
txn.execute(sql_asset)
result_asset = txn.fetchall()
vector.append(result_asset)
vector.append(tools_arr)
except:
lo.wfile("\nError in getting asset,please check your database or network connection......")
tvm.executeVector(vector)
tvg=ToolsVectorGenerator()
class Tool:
def exectool(self,tool):
exec tool
return
def getResult(self,tool):
return deferToThread(self.exectool, tool)
to=Tool()
class StateMachine:
def setPriority(self,txn,tup):
temp=[]
arr=[]
for li in tup:
sql2="select tool_dependency from tool_asset_settings where tool_id =%s" %(li[1],)
txn.execute(sql2)
result12 = txn.fetchall()
arr=[]
if result12[0][0]!=None:
tup12=result12[0][0]
arr=(li[0],tup12)
# print "arr.........",arr
if arr in tup:
print "This element is already exist......."
else:
temp.append(arr)
temp.extend(tup)
return tuple(temp)
st=StateMachine()
class ToolsVectorExecutionManager(object):
def executeVector(self,toolsvector):
print "toolsvector================>",toolsvector
if toolsvector:
for tools in toolsvector[1]:
if tools[2] != None:
to.getResult(tools[2])
tvm=ToolsVectorExecutionManager()
class ToolsToExecuteAnalyzer:
def __init__(self,dbpool=None):
self.dbpool = dbpool
self.loopCall = task.LoopingCall(self.myQuery)
def start(self):
print "Started scanner"
self.loopCall.start(3)
def stop(self):
print "Stopping scanner"
self.loopCall.stop()
def myQuery(self):
def interact(txn):
sql="SELECT tool_asset_id,tool_execute_id FROM tool_to_execute where status='0'"
txn.execute(sql)
result=txn.fetchall()
if result:
tool_asset_id=tuple([int(e[0]) for e in result])
tool_execute_id=tuple([int(e[1]) for e in result])
if len(tool_asset_id)>1:
sql1="SELECT asset_id,tool_id FROM tool_in_assets WHERE tool_asset_id IN %s"%(tool_asset_id,)
else:
sql1="SELECT asset_id,tool_id FROM tool_in_assets WHERE tool_asset_id = (%s)"%(tool_asset_id)
txn.execute(sql1)
tup = txn.fetchall()
#dependency check for the selected tool
asset_tool=st.setPriority(txn,tup)
log_date=lo.log_date()
log=log_date+"::priority have been set for the tools......\n"
lo.wfile(log)
#creating group of asset with their tools
res={}
for element in asset_tool:
if element[0] in res:
res[element[0]].append(int(element[1]))
else:
res[int(element[0])] = [int(element[1])]
#Recored deletion from tool_to_execute table
if res!=None and res.keys()!=[]:
for asset_id in res.iterkeys():
if len(tool_execute_id)>1:
sql_del="delete from tool_to_execute where tool_execute_id in %s " %(tool_execute_id,)
else:
sql_del="delete from tool_to_execute where tool_execute_id = %s" %(tool_execute_id)
txn.execute(sql_del)
#New Addition of vector
tvg.generate(txn,res)
# return res
d = self.dbpool.runInteraction(interact)
d.addCallbacks(self.printResult,self.printError)
def printResult(self,res):
print "In printResult after generate...."
def printError(self,error):
print "Got Error: %r" % error
error.printTraceback()
ToolsToExecuteAnalyzer()
if __name__ == '__main__':
from twisted.internet import reactor
dbpool = adbapi.ConnectionPool("MySQLdb", 'localhost', 'test', 'test','test')
s = ToolsToExecuteAnalyzer(dbpool)
reactor.callWhenRunning(s.start)
reactor.addSystemEventTrigger('before','shutdown',s.stop)
reactor.run()
This is my whole code, I just wanted to know how many threads running, means for each tool new thread?
Anyway, thanks Matt for your help..:)
You may also want to take a look at this snippet which provides a ConnectionPool subclass that reconnects on "MySQL server has gone away".
http://www.gelens.org/2009/09/13/twisted-connectionpool-revisited/