I am trying to implement a simple program in which there are several processes that concurrently communicate with each other by sending and receiving messages. In the program, there are 4 participants (each of which corresponds to a process) and communicate with each other as follows:
P1 sends P2 some_message then P2 sends P3 another_message then P3 sends P4 a_message. Based on the messages each participant receives, they perform a specific action.
Obviously, when, for instance, P1 sends P2 a message, P2 is receiving that message from P1, so they are paired.
I have found different approaches none of which are suitable as they seem to be complicated for I am looking for. For example,
Python MPI which has a restriction of "There are not enough slots available in the system". There are a few ways suggested to sort out the issue but the solutions are a bit complicated.
Socket programming which mostly suits server and client scenario. But my program doesn't have a server. I also checked this answer, which is again based on socket programming.
My question is that isn't there any simpler approach than the above ones so that I can implement what I explained? Is it possible to create communication channels in Python fairly similar to the ones in Golang?
This code I wrote a while ago to get to grips with os.pipe - it is self contained but not "minimally reproducible" since I don't have the time to redo it. It uses tkinter Uis to simulate processes and sends and receives data between them. Note that the code was written only for my private purpose.
"""Test run of the use of pipes between processes.
.. processes are control, startup , send and receive.
.. pipes from control to startup and send
.. pipe from startup to send
.. pipe from send to receive
.. startup, user input of run mode
... prompt, timer (seconds) or number of runs
.. send, user input of data
.. receive, display of data received
. each process operates independently of, and in isolation from, the other processes until data is transferred through pipes
"""
# fr read file descriptor
# fw write file descriptor
# wbs write bytes
# snb string length of output filled with 0 to write as header
# bsnb for number of bytes written, needed for read
# maxbuf number of bytes of header, 4 digits, max 9999 characters in a string/byte literal
# onb output number of bytes
# dbs data read in bytes
import tkinter as tk
from os import pipe as ospipe
from os import read as osread
from os import write as oswrite
from os import close as osclose
from datetime import datetime as dt
from time import monotonic as clock
from functools import partial
BG = '#fa4'
TBG = '#fe8'
SndBG = '#f91'
BLK = '#000'
STOP = '#d30'
START = '#0b0'
start = clock()
def timer(halt):
tm = int(clock())
if int(tm - start) > halt:
return True
else: return False
def piperead(r):
maxbuf = 4
onb = osread(r,maxbuf)
oi = int(onb.decode())
dbs = osread(r,oi).decode() # bytes to string
osclose(r)
return dbs
def pipewrite(w,s):
wbs = bytes(s, encoding='utf-8')
snb = str(len(s)).zfill(4)
bsnb = bytes(snb, encoding='utf-8')
wbs = bsnb + wbs
oswrite(w,wbs)
osclose(w)
def setpipe(process, sub=None, vars=None):
fdr, fdw = ospipe()
if sub: process(fdw,proc=(sub,vars))
else: process(fdw)
return piperead(fdr)
class Sloop():
def __init__(sl, pipewrite=None):
sl.fw = pipewrite
sl.w = tk.Tk()
sl.w.geometry('400x200-100+80')
sl.w.overrideredirect(1)
sl.w['bg'] = BG
uifnt = sl.w.tk.call('font', 'create', 'uifnt', '-family','Consolas', '-size',11)
sl.lvb = tk.Button(sl.w, bg=BLK, activebackground=BG, relief='flat', command=sl.stop)
sl.lvb.pack()
sl.lvb.place(width=15,height=15, x=380,y=10)
sl.sndb = tk.Button(sl.w,bg=SndBG,activebackground=BG,fg=TBG, text=chr(11166), command=sl.send)
sl.sndb.pack()
sl.sndb.place(width=25,height=25, x=20,y=160)
sl.tlbl = tk.Label(sl.w,bg=BG, text='write data to send...')
sl.tlbl.pack()
sl.tlbl.place(x=20,y=20)
sl.t = tk.Text(sl.w,bg=TBG)
sl.t.pack()
sl.t.place(width=300,height=100, x=20,y=45)
sl.t.focus_set()
sl.w.mainloop()
def send(sl):
sl.output = sl.t.get('1.0','end')
if sl.output != '\n':
pipewrite(sl.fw,sl.output)
sl.close()
else:
sl.error()
def error(sl):
def _clearlbl(ev):
sl.erlbl.destroy()
sl.erlbl = tk.Label(sl.w,bg=TBG,text='there is nothing to send')
sl.erlbl.pack()
sl.erlbl.place(x=20,y=160)
sl.t.focus_set()
sl.t.bind('<KeyPress>',_clearlbl)
def stop(sl):
pipewrite(sl.fw,'stop')
sl.close()
def close(sl):
sl.w.destroy()
class Rloop():
def __init__(rl, pipefread=None):
rl.fr = pipefread
rl.w = tk.Tk()
rl.w.geometry('400x200-100+320')
rl.w.overrideredirect(1)
rl.w['bg'] = BG
uifnt = rl.w.tk.call('font', 'create', 'uifnt', '-family','Consolas', '-size',10)
rl.lvb = tk.Button(rl.w, bg=BLK, activebackground=BG, relief='flat', command=rl.close)
rl.lvb.pack()
rl.lvb.place(width=15,height=15, x=380,y=10)
rl.tlbl = tk.Label(rl.w,bg=BG, text='received...')
rl.tlbl.pack()
rl.tlbl.place(x=20,y=20)
rl.t = tk.Text(rl.w,bg=TBG)
rl.t['font'] = uifnt
rl.t.pack()
rl.t.place(width=300,height=100, x=20,y=45)
rl.t.focus_set()
rl.receive()
rl.w.mainloop()
def receive(rl):
rec = piperead(rl.fr)
if rec != 'stop':
rl.t.insert('end', '\n'.join([str(dt.now()), rec]))
else: rl.close()
def close(rl):
rl.w.destroy()
class Startup():
def __init__(su, pipefwrite=None):
su.fw = pipefwrite
su.mode = ''
su.w = tk.Tk()
su.w.geometry('400x200-100+500')
su.w.overrideredirect(1)
su.w['bg'] = BG
uifnt = su.w.tk.call('font', 'create', 'uifnt', '-family','Consolas', '-size',11)
su.lvb = tk.Button(su.w, bg=BLK, activebackground=BG, relief='flat', command=su.stop)
su.lvb.pack()
su.lvb.place(width=15,height=15, x=380,y=10)
su.sndb = tk.Button(su.w,bg=SndBG,activebackground=BG,fg=TBG, text=chr(11166), command=su.send)
su.sndb.pack()
su.sndb.place(width=25,height=25, x=20,y=160)
su.title = tk.Label(su.w,bg=BG, text='Modes to continue data input')
su.title.pack()
su.titley = 10
su.title.place(x=20,y=su.titley)
su.ysp = 20
su.margin = 200
ptxt = 'prompt'
su.pb = tk.Button(su.w,bg=BG, activebackground=BG, text=ptxt, relief='flat', cursor='hand2', command=partial(su._get,e=None, nm='su.pb', ent=None))
tmtxt = ' timer '
su.tmb = tk.Button(su.w,bg=BG, activebackground=BG, text=tmtxt, relief='flat', cursor='hand2', command=partial(su._enter,ent='su.tmb'))
rntxt = ' runs '
su.rnb = tk.Button(su.w,bg=BG, activebackground=BG, text=rntxt, relief='flat', cursor='hand2', command=partial(su._enter,ent='su.rnb'))
su.pb.pack()
su.pby = su.titley + 1.5*su.ysp
su.pb.place(x=25,y=su.pby)
su.tmb.pack()
su.tmby = su.pby + 2*su.ysp
su.tmb.place(x=25,y=su.tmby)
su.rnb.pack()
su.rnby = su.pby + 4*su.ysp
su.rnb.place(x=25,y=su.rnby)
su.formd = {'su.pb':su.pb, 'su.tmb':su.tmb, 'su.rnb':su.rnb}
su.w.mainloop()
def _able(su,nm):
for key in su.formd:
if nm[0:4] not in key:
su.formd[key]['state'] = 'disabled'
else:
su.formd[key]['state'] = 'normal'
def _enter(su,ent):
if ent == 'su.tmb':
tmtxt = 'seconds'
su.tmlbl = tk.Label(su.w,bg=BG, text=tmtxt)
su.tment = tk.Entry(su.w,bg=TBG)
su.tmlbl.pack()
su.tment.pack()
tmlbly = su.tmby
su.tmlbl.place(x=su._margin(tmtxt), y=tmlbly)
su.tment.place(x=su.margin, y=tmlbly)
su.tment.focus_set()
su.tment.bind('<Return>', partial(su._get,nm='su.tment', ent=su.tment))
su.formd = su.formd | {'su.tmlbl':su.tmlbl, 'su.tment':su.tment}
elif ent == 'su.rnb':
rntxt = 'number'
su.rnlbl = tk.Label(su.w,bg=BG, text=rntxt)
su.rnent = tk.Entry(su.w,bg=TBG)
su.rnlbl.pack()
su.rnent.pack()
rnlbly = su.rnby
su.rnlbl.place(x=su._margin(rntxt), y=rnlbly)
su.rnent.place(x=su.margin, y=rnlbly)
su.rnent.focus_set()
su.rnent.bind('<Return>', partial(su._get,nm='su.rnent', ent=su.rnent))
su.formd = su.formd | {'su.rnlbl':su.rnlbl, 'su.rnent':su.rnent}
def _get(su,e,nm,ent):
if nm == 'su.pb':
su._able('su.pb')
su.mode = 'prompt,'+'1'
else:
su._able(nm)
for key in su.formd:
if key == nm:
if 'tm' in key: modestr = 'timer'
elif 'rn' in key: modestr = 'runs'
su.formd[key]['bg']=BG
su.mode = ','.join([modestr,str(ent.get())])
break
def _margin(su,txt):
return su.margin-(len(txt)*8)
def send(su):
pipewrite(su.fw,su.mode)
su.close()
def stop(su):
pipewrite(su.fw,'stop')
su.close()
def close(su):
su.w.destroy()
class Control():
def __init__(c, pipefwrite=None, proc=None):
c.fw = pipefwrite
c.proc = proc
if c.proc:
c.proc = proc[0]
if proc[1]:
c.procv = proc[1]
else:
c.procvl = None
c.procd = {'start':c._strtui, 'prompt':c._prui, 'timer':c._tmui, 'runs':c._rnui}
c.w = tk.Tk()
c.w.geometry('100x200-60+80')
c.w.overrideredirect(1)
c.w['bg'] = BG
uifnt = c.w.tk.call('font', 'create', 'uifnt', '-family','Consolas', '-size',11)
c.lvb = tk.Button(c.w, bg=BLK, activebackground=BG, relief='flat', command=c.stop)
c.lvb.pack()
c.lvb.place(width=15,height=15, x=80,y=10)
c.title = tk.Label(c.w,bg=BG, text='pipe test\nControl')
c.title.pack()
c.title.place(x=5,y=5)
c.stpclr = tk.Label(c.w,bg=STOP)
c.stpclr.pack()
stpy = 160
c.stpclr.place(width=7,height=7,x=2,y=stpy+10)
c.stopb = tk.Button(c.w, bg=BG, text='stop', cursor='hand2', relief='flat', activebackground=BG, command=c.stop)
c.stopb.pack()
c.stopb.place(x=10,y=160)
c.procd[c.proc]()
c.w.mainloop()
def _strtui(c):
c.strtclr = tk.Label(c.w,bg=START)
c.strtclr.pack()
strty = 60
c.strtclr.place(width=7,height=7,x=2,y=strty+10)
c.startb = tk.Button(c.w, bg=BG, text='start', cursor='hand2', relief='flat', activebackground=BG, command=c.strtup)
c.startb.pack()
c.startb.place(x=10,y=strty)
def __write(c,s):
pipewrite(c.fw,s)
c.close()
def _prui(c):
prb = tk.Button(c.w,bg=TBG, text='--- next ---', activebackground=BG, relief='flat',cursor='hand2', command=partial(c.__write,'prompt'))
prb.pack()
prb.place(x=10,y=80)
def __confirm(c):
cb = tk.Button(c.w, bg=TBG, text='confirm', activebackground=BG, relief= 'flat', cursor='hand2', command=partial(c.__write,'confirm'))
cb.pack()
cb.place(x=20,y=120)
def _tmui(c):
tmt = ''.join(['run for\n',str(c.procv),' seconds'])
tmlbl = tk.Label(c.w,bg=BG, text=tmt)
tmlbl.pack()
tmlbl.place(x=10,y=80)
c.__confirm()
def _rnui(c):
rnt = ''.join(['run\n ',str(c.procv),' times'])
rnlbl = tk.Label(c.w,bg=BG, text=rnt)
rnlbl.pack()
rnlbl.place(x=10,y=80)
c.__confirm()
def strtup(c):
pipewrite(c.fw,'startup')
c.close()
def stop(c):
pipewrite(c.fw,'stop')
c.close()
def close(c):
c.w.destroy()
def once():
fr, fw = ospipe()
Sloop(fw)
Rloop(fr)
def many(mkey,mint=1):
"""modes are ('prompt',1), ('timer',secs), ('runs',runs)
"""
if mkey == 'timer':
rec = setpipe(Control,sub='timer',vars=mint)
if rec == 'confirm':
while not timer(mint):
once()
return True
elif rec == 'stop':
return False
elif mkey == 'runs':
rec = setpipe(Control,sub='runs',vars=mint)
if rec == 'confirm':
for r in range(mint):
once()
return True
elif rec == 'stop':
return False
elif mkey == 'prompt':
quit = False
while not quit:
once()
rec = setpipe(Control,sub='prompt')
if rec != 'prompt':
quit = True
return True
def testui():
incontrol = True
while incontrol:
rec = setpipe(Control,sub='start')
if rec == 'startup':
rec = setpipe(Startup)
if rec != 'stop':
modes, p, ns = rec.partition(',')
incontrol = many(modes,int(ns))
else:
incontrol = False
if __name__ == '__main__':
testui()
Related
I am trying to build a command in python for Maya following a course on youtube and it's showing this error "# Error: RuntimeError: file line 2: AttributeError: file C:/Users/saeed/OneDrive/Documents/maya/2020/Plugins/vertexParticle.py line 23: 'module' object has no attribute 'MStatus' # "
I checked Maya API documents and we should have "MStatus" class but I have no idea why it's not accepted, I tried to use "MStatusCode" and it's showing same error.
from maya import OpenMaya
from maya import OpenMayaMPx
from maya import OpenMayaFX
import sys
commandName = "vertexParticle"
kHelpFlag = "-h"
kHelpLongFlag = "-help"
kSparseFlag = "-s"
kSparseLongFlag = "-sparse"
helpMessage = "This command will attach a particle for each vertex"
class pluginCommand(OpenMayaMPx.MPxCommand):
sparse = None
def __init__(self):
OpenMayaMPx.MPxCommand.__init__(self)
def argumentParser(self, argList):
syntax = self.syntax()
parserArguments = OpenMaya.MArgDatabase(syntax, argList)
if parserArguments.isFlagSet(kSparseFlag):
self.sparse = parserArguments.flagArgumentDouble(kSparseFlag, 0)
return OpenMaya.MStatus.kSuccess
if parserArguments.isFlagSet(kSparseLongFlag):
self.sparse = parserArguments.flagArgumentDouble(kSparseLongFlag, 0)
return OpenMaya.MStatus.kSuccess
if parserArguments.isFlagSet(kHelpFlag):
self.setResult(helpMessage)
return OpenMaya.MStatus.kSuccess
if parserArguments.isFlagSet(kHelpLongFlag):
self.setResult(helpMessage)
return OpenMaya.MStatus.kSuccess
def isUndoable(self):
return True
def undoIt(self):
print "undo"
mFnDagNode = OpenMaya.MFnDagNode(self.mObjParticle)
mDagMod = OpenMaya.MDagModifier()
mDagMod.deleteNode(mFnDagNode.parent(0))
mDagMod.doIt()
return OpenMaya.MStatus.kSuccess
def redoIt(self):
mSel = OpenMaya.MSelectionList()
mDagPath = OpenMaya.MDagPath()
mFnMesh = OpenMaya.MFnMesh()
OpenMaya.MGlobal.getActiveSelectionList(mSel)
if mSel.length() >= 1:
try:
mSel.getDagPath(0, mDagPath)
mFnMesh.setObject(mDagPath)
except:
print "please select a poly mesh"
return OpenMaya.MStatus.kUnknownParameter
else:
print "please select a poly mesh"
return OpenMaya.MStatus.kUnknownParameter
mPointArray = OpenMaya.MPointArray()
mFnMesh.getPoints(mPointArray, OpenMaya.MSpace.kWorld)
mFnParticle = OpenMayaFX.MFnParticleSystem()
self.mObjParticle = mFnParticle.create()
mFnParticle = OpenMayaFX.MFnParticleSystem(self.mObjParticle)
counter == 0
for i in xrange(mPointArray.length()):
if i%self.sparse == 0:
mFnParticle.emit(mPointArray[i])
counter += 1
print "total points :" + str(counter)
mFnParticle.saveInitialState()
return OpenMaya.MStatus.kSuccess
def doIt(self, argList):
self.argumentParser(argList)
if self.sparse != None:
self.redoIt()
return OpenMaya.MStatus.kSuccess
def cmdCreator():
return OpenMayaMPx.asMPxPtr(pluginCommand())
def syntaxCreator():
syntax = OpenMaya.MSyntax()
syntax.addFlag(kHelpFlag, kHelpLongFlag)
syntax.addFlag(kSparseFlag, kSparseLongFlag, OpenMaya.MSyntax.kDouble)
return syntax
def initializePlugin(mObject):
mPlugin = OpenMayaMPx.MFnPlugin(mObject)
try:
mPlugin.registerCommand(commandName, cmdCreator, syntaxCreator)
except:
sys.stderr.write("Failed to register" + commandName)
def uninitializePlugin(mObject):
mPlugin = OpenMayaMPx.MFnPlugin(pluginCommand())
try:
mPlugin.deregisterCommand(commandName)
except:
sys.stderr.write("Failed to deregister" + commandName)
Because MStatus.MStatusCode is an enum, you can return its integer value in Python. The only issue is that because in C++ this is an enum, there is no guarantee that the values won't change/shift between releases. This enum has remained consistent since 2019, so there isn't much risk of this happening for MStatus.MStatusCode.
https://help.autodesk.com/view/MAYAUL/2019/ENU/?guid=__cpp_ref_class_m_status_html
https://help.autodesk.com/view/MAYAUL/2023/ENU/?guid=Maya_SDK_cpp_ref_class_m_status_html
Somewhere in the top of your file simply add the constants you intend to use:
MStatus_kSuccess = 0
MStatus_kFailure = 1
MStatus_kUnknownParameter = 5
Then return the constant instead in your functions:
if parserArguments.isFlagSet(kHelpFlag):
self.setResult(helpMessage)
return MStatus_kSuccess
i have a problem with drop table after finish my scenario
when I try to delete using db.drop_table(Routes, if_exists=False, with_all_data=False) or db.drop_table(Routes, if_exists=True, with_all_data=True)
i get an error
pony.orm.core.TransactionError: #db_session-decorated drop_table() function with ddl option cannot be called inside of another db_sessio
i try to delete using Routes.delete() and I got the same one
i read more documentation and i dont know how can i drop this table
my full code
import logging
import random
import requests
import vk_api
from pony.orm import db_session, rollback
from vk_api.bot_longpoll import VkBotLongPoll, VkBotEventType
import handlers
from models import UserState, Registration, Routes, db
try:
import settings
except ImportError:
exit('DO CP settings.py.default setting.py and set token')
log = logging.getLogger('bot')
def configure_logging():
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(logging.Formatter('%(levelname)s %(message)s'))
stream_handler.setLevel(logging.INFO)
log.addHandler(stream_handler)
file_handler = logging.FileHandler('bot.log')
file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
file_handler.setLevel(logging.DEBUG)
log.addHandler(file_handler)
class Bot:
"""
Echo bot for vk.com
Use python 3.7
"""
def __init__(self, group_id, token):
"""
:param group_id:group_id from group vk.com
:param token:secret key
"""
self.group_id = group_id
self.token = token
self.vk = vk_api.VkApi(token=self.token)
self.long_poller = VkBotLongPoll(self.vk, self.group_id)
self.api = self.vk.get_api()
def run(self):
"""run bot"""
for event in self.long_poller.listen():
try:
print(event)
self.on_event(event)
except Exception:
log.exception('ошибка в обработке события')
#db_session
def on_event(self, event):
"""
sends the message back, if message text
:param event: VKBotMessageEvent object
:return: None
"""
if event.type != VkBotEventType.MESSAGE_NEW:
log.info('пока не умеем работать с таким типом события %s', event.type)
return
user_id = event.message.peer_id
text = event.message.text
state = UserState.get(user_id=str(user_id))
if state is not None:
self.continue_scenario(text=text, state=state, user_id=user_id)
else:
# search intent, the user is found outside of the scenario
for intent in settings.INTENTS:
log.debug(f'User gets {intent}')
if any(token in text.lower() for token in intent['tokens']):
# run intent
if intent['answer']:
self.send_text(text_to_send=intent['answer'], user_id=user_id)
else:
self.start_scenario(scenario_name=intent['scenario'], user_id=user_id, text=text)
break
else:
self.send_text(text_to_send=settings.DEFAULT_ANSWER, user_id=user_id)
# функция отправки текста
def send_text(self, text_to_send, user_id):
self.api.messages.send(
message=text_to_send,
random_id=random.randint(0, 2 ** 20),
peer_id=user_id)
# функция отправки изображения
def send_image(self, image, user_id):
upload_url = self.api.photos.getMessagesUploadServer()['upload_url']
upload_data = requests.post(url=upload_url, files={'photo': ('image.png', image, 'image/png')}).json()
image_data = self.api.photos.saveMessagesPhoto(**upload_data)
owner_id = image_data[0]['owner_id']
media_id = image_data[0]['id']
attachment = f'photo{owner_id}_{media_id}'
self.api.messages.send(
attachment=attachment,
random_id=random.randint(0, 2 ** 20),
peer_id=user_id)
# функция отправки маршрутов
def send_routes(self, text_to_send, user_id):
self.api.messages.send(
message=text_to_send,
random_id=random.randint(0, 2 ** 20),
peer_id=user_id)
# шаг отправки
def send_step(self, step, user_id, text, context):
if 'text' in step:
self.send_text(text_to_send=step['text'].format(**context), user_id=user_id)
if 'image' in step:
handler = getattr(handlers, step['image'])
image = handler(text=text, context=context)
self.send_image(image=image, user_id=user_id)
if 'choice' in step:
handler = getattr(handlers, step['choice'])
choices = handler(text=text, context=context)
for i, choice in enumerate(choices, start=1):
city_from = context['city_from']
city_to = context['city_to']
flight_time = choice[city_from][city_to][0]
flight_board = choice[city_from][city_to][1]
flight_date = choice[city_from][city_to][2]
number_choice = str(i) + ') '
Routes(number_choice=number_choice, flight_time=flight_time, city_from=city_from, city_to=city_to,
flight_board=flight_board, flight_date=flight_date)
select = db.select('number_choice,flight_time,city_from,city_to,flight_board,flight_date FROM Routes')
text_to_send = ' '.join(select[0]) + '\n' + ' '.join(select[1]) + '\n' + ' '.join(
select[2]) + '\n' + ' '.join(
select[3]) + '\n' + ' '.join(select[4]) + '\n'
self.send_routes(text_to_send=text_to_send, user_id=user_id)
# начало сценария
def start_scenario(self, scenario_name, user_id, text):
scenario = settings.SCENARIOS[scenario_name]
first_step = scenario['first_step']
step = scenario['steps'][first_step]
self.send_step(step=step, user_id=user_id, text=text, context={})
UserState(user_id=str(user_id), scenario_name=scenario_name, step_name=first_step, context={})
# конец сценария
def continue_scenario(self, text, state, user_id):
steps = settings.SCENARIOS[state.scenario_name]['steps']
step = steps[state.step_name]
handler = getattr(handlers, step['handler'])
if handler(text=text, context=state.context):
# next_step
if step['handler'] != 'handler_answer':
self.walking_through_scenario(state=state, step=step, steps=steps, text=text, user_id=user_id)
else:
if state.context['answer'] == 'да':
self.walking_through_scenario(state=state, step=step, steps=steps, text=text, user_id=user_id)
else:
next_step = step['failure_step']
state.delete()
self.send_step(step=step, user_id=user_id, text=text, context={})
UserState(user_id=str(user_id), scenario_name=state.scenario_name, step_name=next_step, context={})
else:
# retry_current_step
text_to_send = step['failure_text'].format(**state.context)
self.send_text(text_to_send=text_to_send, user_id=user_id)
# движения по сценарию
def walking_through_scenario(self, state, step, steps, text, user_id):
next_step = steps[step['next_step']]
self.send_step(step=next_step, user_id=user_id, text=text, context=state.context)
if next_step['next_step']:
# switch to next step
state.step_name = step['next_step']
else:
# finish scenario and delete user states
log.info(
'Зарегистрирован пользователь ФИО {name} следует из {city_from} в {city_to} телефон для контактов {phone} '.format(
**state.context))
Registration(name=state.context['name'], city_from=state.context['city_from'],
city_to=state.context['city_to'],
phone=state.context['phone'])
state.delete()
if __name__ == '__main__':
configure_logging()
bot = Bot(group_id=settings.GROUP_ID, token=settings.TOKEN)
bot.run()
it helped me this command Routes.select(lambda p: p.id > 0 ).delete(bulk=True)
I have 3 files lcdtest.py, lcd.py and alarmfunctionr.py.
I am trying to control the attached lcd display on my raspberry pi with the lcdtest.py script.
!/usr/bin/env python
import paho.mqtt.client as paho
import globals
import time
from alarmfunctionsr import SendToLCD
from lcd import noDisplay
from lcd import message
globals.init()
SendToLCD(12, "test lcd" ,1) #Test
time.sleep(5)
lcd.message("test with message")
time.sleep(5)
noDisplay
The import from alarmfunctionsr seem to work ok but i get an cannot import name error when i try the same for the lcd script.
lcd.py:
#!/usr/bin/python
#
# based on code from lrvick and LiquidCrystal
# lrvic - https://github.com/lrvick/raspi-hd44780/blob/master/hd44780.py
# LiquidCrystal - https://github.com/arduino/Arduino/blob/master/libraries/LiquidCrystal/LiquidCrystal.cpp
#
from time import sleep
class CharLCD(object):
# commands
LCD_CLEARDISPLAY = 0x01
LCD_RETURNHOME = 0x02
LCD_ENTRYMODESET = 0x04
LCD_DISPLAYCONTROL = 0x08
LCD_CURSORSHIFT = 0x10
LCD_FUNCTIONSET = 0x20
LCD_SETCGRAMADDR = 0x40
LCD_SETDDRAMADDR = 0x80
# flags for display entry mode
LCD_ENTRYRIGHT = 0x00
LCD_ENTRYLEFT = 0x02
LCD_ENTRYSHIFTINCREMENT = 0x01
LCD_ENTRYSHIFTDECREMENT = 0x00
# flags for display on/off control
LCD_DISPLAYON = 0x04
LCD_DISPLAYOFF = 0x00
LCD_CURSORON = 0x02
LCD_CURSOROFF = 0x00
LCD_BLINKON = 0x01
LCD_BLINKOFF = 0x00
# flags for display/cursor shift
LCD_DISPLAYMOVE = 0x08
LCD_CURSORMOVE = 0x00
# flags for display/cursor shift
LCD_DISPLAYMOVE = 0x08
LCD_CURSORMOVE = 0x00
LCD_MOVERIGHT = 0x04
LCD_MOVELEFT = 0x00
# flags for function set
LCD_8BITMODE = 0x10
LCD_4BITMODE = 0x00
LCD_2LINE = 0x08
LCD_1LINE = 0x00
LCD_5x10DOTS = 0x04
LCD_5x8DOTS = 0x00
def __init__(self, pin_rs=25, pin_e=24, pins_db=[23, 17, 27, 22], GPIO=None):
# Emulate the old behavior of using RPi.GPIO if we haven't been given
# an explicit GPIO interface to use
if not GPIO:
import RPi.GPIO as GPIO
GPIO.setwarnings(False)
self.GPIO = GPIO
self.pin_rs = pin_rs
self.pin_e = pin_e
self.pins_db = pins_db
self.GPIO.setmode(GPIO.BCM)
self.GPIO.setup(self.pin_e, GPIO.OUT)
self.GPIO.setup(self.pin_rs, GPIO.OUT)
for pin in self.pins_db:
self.GPIO.setup(pin, GPIO.OUT)
self.write4bits(0x33) # initialization
self.write4bits(0x32) # initialization
self.write4bits(0x28) # 2 line 5x7 matrix
self.write4bits(0x0C) # turn cursor off 0x0E to enable cursor
self.write4bits(0x06) # shift cursor right
self.displaycontrol = self.LCD_DISPLAYON | self.LCD_CURSOROFF | self.LCD_BLINKOFF
self.displayfunction = self.LCD_4BITMODE | self.LCD_1LINE | self.LCD_5x8DOTS
self.displayfunction |= self.LCD_2LINE
# Initialize to default text direction (for romance languages)
self.displaymode = self.LCD_ENTRYLEFT | self.LCD_ENTRYSHIFTDECREMENT
self.write4bits(self.LCD_ENTRYMODESET | self.displaymode) # set the entry mode
self.clear()
def begin(self, cols, lines):
if (lines > 1):
self.numlines = lines
self.displayfunction |= self.LCD_2LINE
def home(self):
self.write4bits(self.LCD_RETURNHOME) # set cursor position to zero
self.delayMicroseconds(3000) # this command takes a long time!
def clear(self):
self.write4bits(self.LCD_CLEARDISPLAY) # command to clear display
self.delayMicroseconds(3000) # 3000 microsecond sleep, clearing the display takes a long time
def setCursor(self, col, row):
self.row_offsets = [0x00, 0x40, 0x14, 0x54]
self.write4bits(self.LCD_SETDDRAMADDR | (col + self.row_offsets[row]))
def noDisplay(self):
""" Turn the display off (quickly) """
self.displaycontrol &= ~self.LCD_DISPLAYON
self.write4bits(self.LCD_DISPLAYCONTROL | self.displaycontrol)
def display(self):
""" Turn the display on (quickly) """
self.displaycontrol |= self.LCD_DISPLAYON
self.write4bits(self.LCD_DISPLAYCONTROL | self.displaycontrol)
def noCursor(self):
""" Turns the underline cursor off """
self.displaycontrol &= ~self.LCD_CURSORON
self.write4bits(self.LCD_DISPLAYCONTROL | self.displaycontrol)
def cursor(self):
""" Turns the underline cursor on """
self.displaycontrol |= self.LCD_CURSORON
self.write4bits(self.LCD_DISPLAYCONTROL | self.displaycontrol)
def noBlink(self):
""" Turn the blinking cursor off """
self.displaycontrol &= ~self.LCD_BLINKON
self.write4bits(self.LCD_DISPLAYCONTROL | self.displaycontrol)
def blink(self):
""" Turn the blinking cursor on """
self.displaycontrol |= self.LCD_BLINKON
self.write4bits(self.LCD_DISPLAYCONTROL | self.displaycontrol)
def DisplayLeft(self):
""" These commands scroll the display without changing the RAM """
self.write4bits(self.LCD_CURSORSHIFT | self.LCD_DISPLAYMOVE | self.LCD_MOVELEFT)
def scrollDisplayRight(self):
""" These commands scroll the display without changing the RAM """
self.write4bits(self.LCD_CURSORSHIFT | self.LCD_DISPLAYMOVE | self.LCD_MOVERIGHT)
def leftToRight(self):
""" This is for text that flows Left to Right """
self.displaymode |= self.LCD_ENTRYLEFT
self.write4bits(self.LCD_ENTRYMODESET | self.displaymode)
def rightToLeft(self):
""" This is for text that flows Right to Left """
self.displaymode &= ~self.LCD_ENTRYLEFT
self.write4bits(self.LCD_ENTRYMODESET | self.displaymode)
def autoscroll(self):
""" This will 'right justify' text from the cursor """
self.displaymode |= self.LCD_ENTRYSHIFTINCREMENT
self.write4bits(self.LCD_ENTRYMODESET | self.displaymode)
def noAutoscroll(self):
""" This will 'left justify' text from the cursor """
self.displaymode &= ~self.LCD_ENTRYSHIFTINCREMENT
self.write4bits(self.LCD_ENTRYMODESET | self.displaymode)
def write4bits(self, bits, char_mode=False):
""" Send command to LCD """
self.delayMicroseconds(1000) # 1000 microsecond sleep
bits = bin(bits)[2:].zfill(8)
self.GPIO.output(self.pin_rs, char_mode)
for pin in self.pins_db:
self.GPIO.output(pin, False)
for i in range(4):
if bits[i] == "1":
self.GPIO.output(self.pins_db[::-1][i], True)
self.pulseEnable()
for pin in self.pins_db:
self.GPIO.output(pin, False)
for i in range(4, 8):
if bits[i] == "1":
self.GPIO.output(self.pins_db[::-1][i-4], True)
self.pulseEnable()
def delayMicroseconds(self, microseconds):
seconds = microseconds / float(1000000) # divide microseconds by 1 million for seconds
sleep(seconds)
def pulseEnable(self):
self.GPIO.output(self.pin_e, False)
self.delayMicroseconds(1) # 1 microsecond pause - enable pulse must be > 450ns
self.GPIO.output(self.pin_e, True)
self.delayMicroseconds(1) # 1 microsecond pause - enable pulse must be > 450ns
self.GPIO.output(self.pin_e, False)
self.delayMicroseconds(1) # commands need > 37us to settle
def message(self, text):
""" Send string to LCD. Newline wraps to second line"""
for char in text:
if char == '\n':
self.write4bits(0xC0) # next line
else:
self.write4bits(ord(char), True)
def DisplayLCD(msg):
lcd = CharLCD()
lcd.clear()
x=msg.find("**")
if x>0:
line1=msg[0:x]
line2=msg[x+2:len(msg)]
else:
line1=msg
line2=""
lcd.message(line1+"\n"+line2)
alarmfunctionsr.py:
#!/usr/bin/env python
"""
import globals
import urllib2
import smtplib
import serial
import time
import sys
import thread
import RPi.GPIO as GPIO
import os, glob, time, operator
from email.mime.text import MIMEText
from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from time import sleep
def find_all(a_str, sub):
start = 0
cnt=0
while True:
start = a_str.find(sub, start)
if start == -1:
return cnt
start += len(sub)
cnt=cnt+1
def isNumber(x):
# Test whether the contents of a string is a number
try:
val = int(x)
except ValueError:
return False
return True
def get_latest_photo(files):
lt = operator.lt
if not files:
return None
now = time.time()
latest = files[0], now - os.path.getctime(files[0])
for f in files[1:]:
age = now - os.path.getctime(f)
if lt(age, latest[1]):
latest = f, age
return latest[0]
def UpdateHostThread(function,opcode):
try:
thread.start_new_thread(UpdateHostThread, (function,opcode, ) )
except:
print "Error: unable to start thread"
def UpdateHost(function,opcode):
# Sends data to the server
script_path = "https://www.privateeyepi.com/alarmhostr.php?u="+globals.user+"&p="+globals.password+"&function="+str(function)
i=0
for x in opcode:
script_path=script_path+"&opcode"+str(i)+"="+str(opcode[i])
i=i+1
if globals.PrintToScreen: print "Host Update: "+script_path
try:
rt=urllib2.urlopen(script_path)
except urllib2.HTTPError:
if globals.PrintToScreen: print "HTTP Error"
return False
time.sleep(.2)
temp=rt.read()
if globals.PrintToScreen: print temp
l = find_all(temp,"/n");
RecordSet = temp.split(',')
c=[]
y=0
c.append([])
for x in RecordSet:
if x=="/n":
y=y+1
if y < l:
c.append([])
else:
if isNumber(x):
c[y].append(int(x))
else:
c[y].append(x)
rt=ProcessActions(c)
if rt==False:
return(False)
else:
return(c)
def ProcessActions(ActionList):
FalseInd=True
for x in ActionList:
if x[0]=="/EMAIL":
SendEmailAlertFromRule(x[1], x[2],0)
x.remove
if x[0]=="/SEMAIL":
SendEmailAlert(x[1])
x.remove
if x[0]=="/CHIME":
StartChimeThread()
x.remove
if x[0]=="/rn588":
exit()
if x[0]=="/FALSE":
FalseInd=False
if x[0]=="/SIREN":
StartSirenThread(x[2])
x.remove
if x[0]=="/PHOTO":
SendEmailAlertFromRule(x[1], x[2],1)
x.remove
if x[0]=="/RELAYON":
SwitchRelay(1)
x.remove
if x[0]=="/RELAYOFF":
SwitchRelay(0)
x.remove
if x[0]=="/WRELAYON":
SwitchRFRelay(1)
x.remove
if x[0]=="/WRELAYOFF":
SwitchRFRelay(0)
x.remove
return(FalseInd)
def StartSirenThread(Zone):
try:
thread.start_new_thread(Siren, (Zone, ) )
except:
print "Error: unable to start thread"
def SwitchRelay(onoff):
GPIO.setmode(GPIO.BOARD)
GPIO.setup(globals.RelayPin, GPIO.OUT)
GPIO.output(globals.RelayPin,onoff)
def SwitchRFRelay(onoff):
# declare to variables, holding the com port we wish to talk to and the speed
port = '/dev/ttyAMA0'
baud = 9600
# open a serial connection using the variables above
ser = serial.Serial(port=port, baudrate=baud)
# wait for a moment before doing anything else
sleep(0.2)
for i in range(0,3):
if (onoff==True):
ser.write('a{}RELAYAON-'.format(globals.WRelayPin))
else:
ser.write('a{}RELAYAOFF'.format(globals.WRelayPin))
time.sleep(2)
ser.close
def SendToLCD(GPIOnumber, Location, status):
import paho.mqtt.client as paho
if status==0:
ActionStr="_"
topic="alarm_activity"
else:
if status==1:
ActionStr="_"
topic="alarm_activity"
else:
topic="temperature"
if status==2:
ActionStr=str(GPIOnumber)+","+Location
else:
ActionStr="Undefined"
#client = mosquitto.Mosquitto('privateeyepi')
client = paho.Client()
client.connect(globals.lcd_ip)
if status <= 1:
if globals.PrintToScreen:
print str(Location)+"**"+str(ActionStr)
client.publish(topic, str(Location)+"**"+str(ActionStr))
else:
if globals.PrintToScreen:
print str(ActionStr)
client.publish(topic, ActionStr)
client.disconnect()
def Siren(Zone):
GPIO.setmode(GPIO.BOARD)
if globals.UseSiren == True:
GPIO.setup(globals.SirenGPIOPin, GPIO.OUT) #Siren pin setup
else:
return
if globals.SirenDelay>0:
globals.SirenStartTime = time.time()
while time.time() < globals.SirenStartTime + globals.SirenDelay:
if globals.BeepDuringDelay:
GPIO.output(globals.SirenGPIOPin,True)
time.sleep(1)
GPIO.output(globals.SirenGPIOPin,False)
time.sleep(4)
GPIO.output(globals.SirenGPIOPin,True)
globals.SirenStartTime = time.time()
if globals.PrintToScreen: print "Siren Activated"
while time.time() < globals.SirenStartTime + globals.SirenTimeout:
time.sleep(5)
if CheckForSirenDeactivation(Zone) == True:
break
GPIO.output(globals.SirenGPIOPin,False)
if globals.PrintToScreen: print "Siren Deactivated"
def CheckForSirenDeactivation(Zone):
# Routine to fetch the location and zone descriptions from the server
RecordSet = GetDataFromHost(16,[Zone])
if globals.PrintToScreen: print RecordSet
ZoneStatus=RecordSet[0][0]
if ZoneStatus=="FALSE":
return (True)
def StartChimeThread():
try:
thread.start_new_thread(SoundChime, ())
except:
print "Error: unable to start thread"
def SoundChime():
if globals.ChimeDuration>0:
GPIO.setmode(GPIO.BOARD)
GPIO.setup(globals.ChimeGPIOPin, GPIO.OUT) #Siren pin setup
GPIO.output(globals.ChimeGPIOPin,True)
time.sleep(globals.ChimeDuration)
GPIO.output(globals.ChimeGPIOPin,False)
def GetDataFromHost(function,opcode):
# Request data and receive reply (request/reply) from the server
script_path = "https://www.privateeyepi.com/alarmhostr.php?u="+globals.user+"&p="+globals.password+"&function="+str(function)
i=0
for x in opcode:
script_path=script_path+"&opcode"+str(i)+"="+str(opcode[i])
i=i+1
if globals.PrintToScreen: print script_path
try:
rt = urllib2.urlopen(script_path)
except urllib2.HTTPError:
return False
temp=rt.read()
if globals.PrintToScreen: print temp
l = find_all(temp,"/n");
RecordSet = temp.split(',')
c=[]
y=0
c.append([])
for x in RecordSet:
if x=="/n":
y=y+1
if y < l:
c.append([])
else:
if isNumber(x):
c[y].append(int(x))
else:
c[y].append(x)
rt=ProcessActions(c)
if rt==False:
return(False)
else:
return(c)
return(c)
def BuildMessage(SensorNumber):
# Routine to fetch the location and zone descriptions from the server
RecordSet = GetDataFromHost(6,[SensorNumber])
if globals.PrintToScreen: print RecordSet
if RecordSet==False:
return
zonedesc=RecordSet[0][0]
locationdesc = RecordSet[0][1]
messagestr="This is an automated email from your house alarm system. Alarm activated for Zone: "+zonedesc+" ("+locationdesc+")"
return messagestr
def BuildMessageFromRule(SensorNumber, smartruleid):
RecordSet = GetDataFromHost(7,[smartruleid, SensorNumber])
if RecordSet==False:
return
numrows = len(RecordSet)
messagestr="This is an automated email from PrivateEyePi. Rule triggered for Zone(s): "+RecordSet[0][3]+", Location: "+RecordSet[0][4]+" and for rule "
for i in range(0,numrows,1):
if RecordSet[i][0]==1:
messagestr=messagestr+"Alarm Activated"
if RecordSet[i][0]==2:
messagestr=messagestr+"Alarm Deactivated"
if RecordSet[i][0]==3:
messagestr=messagestr+"Circuit Open"
if RecordSet[i][0]==4:
messagestr=messagestr+"Circuit Closed"
if RecordSet[i][0]==5:
messagestr=messagestr+"Open for " + str(RecordSet[i][1]) + " Minutes"
if RecordSet[i][0]==6:
messagestr=messagestr+"Closed for " + str(RecordSet[i][1]) + " Minutes"
if RecordSet[i][0]==7:
messagestr=messagestr+"Where sensor value (" + str(RecordSet[i][5]) + ") is between " + str(RecordSet[i][1]) + " " + str(RecordSet[i][2])
if RecordSet[i][0]==8:
messagestr=messagestr+"Tamper"
if RecordSet[i][0]==9:
messagestr=messagestr+"Day Of Week is between " + str(RecordSet[i][1]) + " and " + str(RecordSet[i][2])
if RecordSet[i][0]==10:
messagestr=messagestr+"Hour Of Day is between " + str(RecordSet[i][1]) + " and " + str(RecordSet[i][2])
if RecordSet[i][0]==11:
messagestr=messagestr+"Where secondary sensor value (" + str(RecordSet[i][6]) + ") is between " + str(RecordSet[i][1]) + " " + str(RecordSet[i][2])
if i<numrows-1:
messagestr=messagestr + " AND "
return messagestr
def SendEmailAlertFromRule(ruleid, SensorNumber, photo):
try:
thread.start_new_thread(SendEmailAlertThread, (SensorNumber, ruleid, True, photo, ) )
except:
print "Error: unable to start thread"
def SendEmailAlert(SensorNumber):
try:
thread.start_new_thread(SendEmailAlertThread, (SensorNumber,0 , False, False) )
except:
print "Error: unable to start thread"
def SendEmailAlertThread(SensorNumber, smartruleid, ruleind, photo):
# Get the email addresses that you configured on the server
RecordSet = GetDataFromHost(5,[0])
if RecordSet==False:
return
numrows = len(RecordSet)
if globals.smtp_server=="":
return
if ruleind:
msgtext = BuildMessageFromRule(SensorNumber, smartruleid)
else:
msgtext = BuildMessage(SensorNumber)
for i in range(numrows):
# Define email addresses to use
addr_to = RecordSet[i][0]
addr_from = globals.smtp_user #Or change to another valid email recognized under your account by your ISP
# Construct email
if (photo==1):
files = 0
files = glob.glob(globals.photopath)
latestphoto = get_latest_photo(files)
msg = MIMEMultipart()
else:
msg = MIMEText(msgtext)
msg['To'] = addr_to
msg['From'] = addr_from
msg['Subject'] = 'Alarm Notification' #Configure to whatever subject line you want
#attach photo
if (photo==1):
msg.preamble = 'Multipart message.\n'
part = MIMEText(msgtext)
msg.attach(part)
part = MIMEApplication(open(latestphoto,"rb").read())
part.add_header('Content-Disposition', 'attachment', filename=latestphoto)
msg.attach(part)
# Send the message via an SMTP server
#Option 1 - No Encryption
if globals.email_type==1:
s = smtplib.SMTP(globals.smtp_server)
elif globals.email_type==2:
#Option 2 - SSL
s = smtplib.SMTP_SSL(globals.smtp_server, 465)
elif globals.email_type==3:
#Option 3 - TLS
s = smtplib.SMTP(globals.smtp_server,587)
s.ehlo()
s.starttls()
s.ehlo()
else:
s = smtplib.SMTP(globals.smtp_server)
s.login(globals.smtp_user,globals.smtp_pass)
s.sendmail(addr_from, addr_to, msg.as_string())
s.quit()
if globals.PrintToScreen: print msg;
def SendCustomEmail(msgText, msgSubject):
# Get the email addresses that you configured on the server
RecordSet = GetDataFromHost(5,[0])
if RecordSet==False:
return
numrows = len(RecordSet)
if globals.smtp_server=="":
return
for i in range(numrows):
# Define email addresses to use
addr_to = RecordSet[i][0]
addr_from = globals.smtp_user #Or change to another valid email recognized under your account by your ISP
# Construct email
msg = MIMEText(msgText)
msg['To'] = addr_to
msg['From'] = addr_from
msg['Subject'] = msgSubject #Configure to whatever subject line you want
# Send the message via an SMTP server
#Option 1 - No Encryption
if globals.email_type==1:
s = smtplib.SMTP(globals.smtp_server)
elif globals.email_type==2:
#Option 2 - SSL
s = smtplib.SMTP_SSL(globals.smtp_server, 465)
elif globals.email_type==3:
#Option 3 - TLS
s = smtplib.SMTP(globals.smtp_server,587)
s.ehlo()
s.starttls()
s.ehlo()
else:
s = smtplib.SMTP(globals.smtp_server)
s.login(globals.smtp_user,globals.smtp_pass)
s.sendmail(addr_from, addr_to, msg.as_string())
s.quit()
if globals.PrintToScreen: print msg;
Your lcd.py module doesn't define any functions (or other top-level objects) named noDisplay or message, only a class named CharLCD and a function named DisplayLCD. So, when you try to import something that doesn't exist, of course you get an ImportError.
It's true that the CharLCD class has methods named noDisplay and message, but that doesn't mean you can just import them as top-level functions. (And, even if you could, you can't call them that way; you need a CharLCD object to call its methods.)
I suspect you need to read a basic tutorial on classes, like the Classes chapter in the official tutorial.
Meanwhile, I think the code you want is:
from lcd import CharLCD
# ...
char_lcd = CharLCD()
char_lcd.message("test with message")
time.sleep(5)
char_lcd.noDisplay()
(Also note the () in the last line. You need those parentheses to call a function or method; without them, you're just referring to the function or method itself, as a value, which has no more effect than just writing 2 on a line by itself.)
I have a script (Django Management-Command) wiht over 800 lines of code.
This should import data from a external Web-Service, manipulate sth. and write it to a Postgres DB.
I use multithreading, because fetching data from webservice ist not very fast.
There ist one Thread for fetching the data with a bulk command to get a bulk of 64 data sets an write each data set in a queue.
Simultaneously at the beginning there is one worker-thread wich manipulates the data and write it to a DB.
In the main (handle) class, there is a while-loop that looks every 5 seconds for the quantity of elements in the queue and the quantity of running worker-threads.
If there are more than 500 elements in the queue and there are less then 5 worker-threads, it starts a new worker-thread.
All worker-threads get one item from the queue, manipulate sth., write the data set to the DB and append one String (up to 14 chars) to a different queue (#2).
The queue #2 ist necessary to have all imported objects at the end of the import to mark them as new respectively delete all other items from the DB, which are currently not imported.
For DB's with a quantity of not more then 200.000 data sets everything works fine.
But if there is for example a DB with 1.000.000 data sets, the memory consumption increases during the processing of the hole script up to 8 GB of RAM.
Is there a method to watch the memory consumption of threads and / or queue's?
Is there a method to "clean" memory after each while-loop?
# -*- coding: utf-8 -*-
import os
import threading
import Queue
import time
from optparse import OptionParser, make_option
from decimal import Decimal
from datetime import datetime
from django.core.management import call_command
from django.core.management.base import BaseCommand
from django.conf import settings
def is_someone_alive(thread_list):
so_alive = False
for t in thread_list:
if t.is_alive():
so_alive = True
return so_alive
class insert_item(threading.Thread):
VarLock2 = threading.Lock()
def __init__(self, queue1, item_still_exist2, name, *args, **options):
threading.Thread.__init__(self)
self.options = options
self.name = name
self.queue1 = queue1
self.item_still_exist2 = item_still_exist2
def run(self):
while not self.queue1.empty() or getItemBulkThread.isrunning:
item = self.queue1.get()
artikelobj, created = Artikel.objects.get_or_create(artikelnr=item['Nr'])
"""
manipulate data
"""
self.item_still_exist2.put(artikelobj.artikelnr)
artikelobj.save()
self.queue1.task_done()
class getItemBulkThread(threading.Thread):
isrunning = True
VarLock = threading.Lock()
def __init__(self, queue1, name, *args, **options):
threading.Thread.__init__(self)
self.options = options
if self.options['nrStart'] != '':
self.nrab = self.options['nrStart']
else:
self.nrab = ''
self.name = name
#self.nrab = '701307'
self.queue1 = queue1
self.anz_artikel = 64
self.max_artikel = 64
self.skipped = 0
self.max_skip = 20
def run(self):
count_sleep = 0
while True:
while self.queue1.qsize() > 5000:
time.sleep(5)
count_sleep += 1
if count_sleep > 0:
print "~ Artikel-Import %(csleep)sx für 5s pausiert, da Queue-Size > 5000" % {'csleep': count_sleep}
count_sleep = 0
try:
items = getItemBulk() # from external service
except Exception as exc1:
if ('"normal" abort-condition' in str(exc1)):
getItemBulkThread.VarLock.acquire()
getItemBulkThread.isrunning = False
getItemBulkThread.VarLock.release()
break
elif self.anz_artikel > 1:
self.anz_artikel /= 2
continue
elif self.skipped <= self.max_skip:
self.nrab += 1
self.skipped += 1
time.sleep(5)
continue
elif self.skipped > self.max_skip:
raise Exception("[EXCEPTION] Fehler im Thread: too much items skipped")
else:
getItemBulkThread.VarLock.acquire()
getItemBulkThread.isrunning = False
getItemBulkThread.VarLock.release()
raise
last_item = len(items) - 1
self.nrab = items[last_item]['Nr']
for artikel in items:
artikel['katItem'] = False
self.queue1.put(artikel)
if self.anz_artikel < self.max_artikel:
self.anz_artikel *= 2
self.skipped = 0
class Command(BaseCommand):
"""
Django-mgm-command
"""
help = u'Import'
def create_parser(self, prog_name, subcommand):
"""
Create and return the ``OptionParser`` which will be used to
parse the arguments to this command.
"""
return OptionParser(prog=prog_name, usage=self.usage(subcommand),
version=self.get_version(),
option_list=self.option_list,
conflict_handler="resolve")
def handle(self, *args, **options):
startzeit = datetime.now()
anzahl_Artikel_vorher = Artikel.objects.all().count() # Artikel is a model
self.options = options
items_vorher = []
queue1 = Queue.Queue()
item_still_exists2 = Queue.Queue()
running_threads = []
thread = getItemBulkThread(queue1, name="Artikel", *args, **options)
running_threads.append(thread)
thread.daemon = True
thread.start()
anz_worker_threads = 1
anz_max_worker_threads = 5
insert_threads = [insert_item(queue1, item_still_exists2, name="Worker-%(anz)s" % {'anz': i + 1}, *args, **options) for i in range(anz_worker_threads)]
for thread in insert_threads:
running_threads.append(thread)
thread.setDaemon(True)
thread.start()
add_seconds = 5
element_grenze = 500
lastelemente = 0
asc_elemente = 0
anz_abgearbeitet = 0
while getItemBulkThread.isrunning or not queue1.empty():
time.sleep(add_seconds)
elemente = queue1.qsize()
akt_zeit = datetime.now()
diff_zeit = akt_zeit - startzeit
diff = elemente - lastelemente
anz_abgearbeitet = item_still_exists2.qsize()
art_speed = (anz_abgearbeitet / timedelta_total_seconds(diff_zeit)) * 60
ersetz_var = {'anz': elemente, 'zeit': diff_zeit, 'tstamp': akt_zeit.strftime('%Y.%m.%d-%H:%M:%S'), 'anzw': anz_worker_threads, 'diff': diff, 'anza': anz_abgearbeitet, 'art_speed': art_speed}
print("%(zeit)s vergangen - %(tstamp)s - %(anz)s Elemente in Queue, Veränderung: %(diff)s - Anz Worker: %(anzw)s - Artikel importiert: %(anza)s - Speed: %(art_speed)02d Art/Min" % ersetz_var)
if diff > 0:
asc_elemente += 1
else:
asc_elemente = 0
if asc_elemente > 2 and anz_worker_threads < anz_max_worker_threads and elemente > element_grenze:
ersetz_var = {'maxw': anz_max_worker_threads, 'nr': anz_worker_threads + 1, 'element_grenze': element_grenze}
print "~~ 2x in Folge mehr Queue-Elemente als vorher, die max. Anzahl an Workern %(maxw)s noch nicht erreicht und mehr als %(element_grenze)s Elemente in der Queue, daher Start eines neuen Workers (Nr %(nr)s)" % ersetz_var
anz_worker_threads += 1
thread = insert_item(queue1, item_still_exists2, name="Worker-%(anz)s" % {'anz': anz_worker_threads}, *args, **options)
running_threads.append(thread)
thread.setDaemon(True)
thread.start()
asc_elemente = 0
lastelemente = elemente
queue1.join()
items_nachher = []
while not item_still_exists2.empty():
item = item_still_exists2.get()
if item in items_vorher:
items_nachher.append(item)
items_vorher.remove(item)
item_still_exists2.task_done()
item_still_exists2.join()
if len(items_vorher) > 0:
Artikel.objects.filter(artikelnr__in=items_vorher).delete()
anzahl_Artikel_nachher = Artikel.objects.all().count()
anzahl_Artikel_diff = anzahl_Artikel_nachher - anzahl_Artikel_vorher
endzeit = datetime.now()
dauer = endzeit - startzeit
I've abbreviated the Code at some positions :)
A possible cause for excessive memory consumption is that you don't set a maximum size for the input queue. See the maxsize parameter.
On a related note, you write:
In the main (handle) class, there is a while-loop that looks every 5
seconds for the quantity of elements in the queue and the quantity of
running worker-threads. If there are more than 500 elements in the
queue and there are less then 5 worker-threads, it starts a new
worker-thread.
Creating a new thread does not necessarily increase the throughput. You should rather do some tests to determine the optimal number of threads, which may turn out to be 1.
I am porting some Java code to Python and we would like to use Python 3 but I can't find LDAP module for Python 3 in Windows.
This is forcing us to use 2.6 version and it is bothersome as rest of the code is already in 3.0 format.
You may use ldap3 module (formerly known as python3-ldap), it runs on python3 really well and requires no external C dependances. Also it can correctly handle both unicode and byte data in ldap records (in early versions there was a trouble with jpegPhoto field, now everything is fine)
If you're running this on Windows, you can get LDAP to work in Python 3.1 by using the ADO access method via Mark Hammond's PyWin32.
To test this, I installed ActiveState Python 3.1, then installed PyWin32 for Python 3.1
http://sourceforge.net/projects/pywin32/files/
I was then able to run LDAP queries using a module I wrote that is based on this LDAP code from the ActiveState Python Cookbook:
Recipe 511451: Dump all Active Directory Information using LDAP scripting by Manuel Garcia
http://code.activestate.com/recipes/511451/
although now that I look at it I realize I completely rewrote my module just using his code as an example.
Update
Here is my LDAPList module and another support module to convert user access bit codes into something a tiny bit more english-like:
LDAPList.py
# LDAPList.py
# Todd Fiske
# class to encapsulate accessing LDAP information
# 2009-03-18 first version
# 2010-01-04 updated for Python 3 (print functions, <> to !=)
import win32com.client
import UACCodes
ADS_SCOPE_SUBTREE = 2
class LDAPList():
def __init__(self, sContext):
self.Context = sContext # naming context, "DC=xyz,DC=org"
self.objectCategory = ""
self.objectClass = ""
self.FilterClause = ""
self.query = ""
self.cn = None
self.cm = None
self.rs = None
def SetCategory(self, sCategory):
self.objectCategory = sCategory
self.FilterClause = "where objectCategory = '%s'" % self.objectCategory
def SetClass(self, sClass):
self.objectClass = sClass
self.FilterClause = "where objectClass = '%s'" % self.objectClass
def open(self):
self.query = "select * from 'LDAP://%s' %s order by displayName" % (self.Context, self.FilterClause)
self.cn = win32com.client.Dispatch("ADODB.Connection")
self.cm = win32com.client.Dispatch("ADODB.Command")
self.cn.Open("Provider=ADsDSOObject")
self.cm.ActiveConnection = self.cn
self.cm.Properties["Page Size"] = 1000
self.cm.Properties["Searchscope"] = ADS_SCOPE_SUBTREE
self.cm.CommandText = self.query
self.rs = self.cm.Execute()[0]
def close(self):
if self.rs is not None:
self.rs.Close()
self.rs = None
if self.cm is not None:
self.cm = None
if self.cn is not None:
self.cn.Close()
self.cn = None
def count(self):
if self.rs is None:
return -2
return self.rs.RecordCount
def more(self):
if self.rs is None:
return False
return not self.rs.EOF
def GetObject(self):
if self.rs is None:
return None
return win32com.client.GetObject(self.rs.Fields["ADsPath"].Value)
def next(self):
if self.rs is None:
return
self.rs.MoveNext()
#----------
# helper functions
def NamingContext():
# return default naming context
root = win32com.client.GetObject("LDAP://RootDse")
return root.get("DefaultNamingContext")
def AccountControl(obj):
if obj.userAccountControl is not None:
return obj.userAccountControl
else:
return 0
def ConvertUAC(nUAC):
return UACCodes.ConvertUAC(nUAC)
def AccountActive(n):
return (n & UACCodes.ADS_UF_ACCOUNTDISABLE) != UACCodes.ADS_UF_ACCOUNTDISABLE
def GetCategory(obj):
# CN=Group,CN=Schema,CN=Configuration,DC=xyz,DC=org
s = obj.objectCategory
s = s.split(",")[0][3:]
return s
# s = "Group"
def GetGroups(obj):
"""
('CN=XYZ Staff Rockville,OU=Distribution Groups,DC=xyz,DC=org',
'CN=XYZ Staff,OU=Distribution Groups,DC=xyz,DC=org')
"""
if obj.memberOf is None:
return ""
if type(obj.memberOf)==type(()):
tGroups = obj.memberOf
else:
tGroups = (obj.memberOf,)
return tGroups
def GetNameParts(obj):
if obj.givenName is None:
sFirst = ""
else:
sFirst = obj.givenName
if obj.middleName is None:
sMiddle = ""
else:
sMiddle = obj.middleName
if obj.sn is None:
sLast = ""
else:
sLast = obj.sn
if sLast == "" and sFirst == "":
if obj.name is not None:
sName = obj.name
sName = sName[3:]
lParts = sName.split(" ")
if len(lParts) == 1:
"todo: split on embedded capital letter"
print("single-part name: %s" % sName)
sFirst = sName
else:
sLast = lParts[-1]
sFirst = " ".join(lParts[:-1])
return (sFirst, sMiddle, sLast)
def GetManager(obj):
if obj.manager is None:
return ""
else:
return obj.manager
#----------
# test
if __name__ == "__main__":
print
print("testing LDAPList class")
nc = NamingContext()
print("context =", nc)
ll = LDAPList(nc)
ll.SetCategory('user')
ll.open() # generates recordset
print("query = %s" % ll.query)
print("%d items" % ll.count())
n = 0
while (n < 10) and (ll.more()):
o = ll.GetObject() # return
nUAC = AccountControl(o)
print("%-30s %-30s %-30s %-40s %s" % (
o.displayName,
o.name,
o.sAMAccountName,
UACCodes.ConvertUAC(nUAC),
GetManager(o)
))
n += 1
ll.next()
ll.close()
###
UACCodes.py
# UACCodes.py
# Todd Fiske
# generated 2009-09-23 16:36:56 by BuildUACCodes.py
# updated 2010-01-04 for Python 3 (print functions)
# provide UAC constants, lookup list, and conversion function
import sys
# UAC Constants
ADS_UF_SCRIPT = 0x00000001
ADS_UF_ACCOUNTDISABLE = 0x00000002
ADS_UF_HOMEDIR_REQUIRED = 0x00000008
ADS_UF_LOCKOUT = 0x00000010
ADS_UF_PASSWD_NOTREQD = 0x00000020
ADS_UF_PASSWD_CANT_CHANGE = 0x00000040
ADS_UF_ENCRYPTED_TEXT_PASSWORD_ALLOWED = 0x00000080
ADS_UF_TEMP_DUPLICATE_ACCOUNT = 0x00000100
ADS_UF_NORMAL_ACCOUNT = 0x00000200
ADS_UF_INTERDOMAIN_TRUST_ACCOUNT = 0x00000800
ADS_UF_WORKSTATION_TRUST_ACCOUNT = 0x00001000
ADS_UF_SERVER_TRUST_ACCOUNT = 0x00002000
ADS_UF_DONT_EXPIRE_PASSWD = 0x00010000
ADS_UF_MNS_LOGON_ACCOUNT = 0x00020000
ADS_UF_SMARTCARD_REQUIRED = 0x00040000
ADS_UF_TRUSTED_FOR_DELEGATION = 0x00080000
ADS_UF_NOT_DELEGATED = 0x00100000
ADS_UF_USE_DES_KEY_ONLY = 0x00200000
ADS_UF_DONT_REQUIRE_PREAUTH = 0x00400000
ADS_UF_PASSWORD_EXPIRED = 0x00800000
ADS_UF_TRUSTED_TO_AUTHENTICATE_FOR_DELEGATION = 0x01000000
# UAC short name lookup list
lUACCodes = [
("ADS_UF_SCRIPT" , 0x00000001, "script"),
("ADS_UF_ACCOUNTDISABLE" , 0x00000002, "disabled"),
("ADS_UF_HOMEDIR_REQUIRED" , 0x00000008, "homedir"),
("ADS_UF_LOCKOUT" , 0x00000010, "lockout"),
("ADS_UF_PASSWD_NOTREQD" , 0x00000020, "pwnotreqd"),
("ADS_UF_PASSWD_CANT_CHANGE" , 0x00000040, "pwcantchange"),
("ADS_UF_ENCRYPTED_TEXT_PASSWORD_ALLOWED" , 0x00000080, "encryptedpw"),
("ADS_UF_TEMP_DUPLICATE_ACCOUNT" , 0x00000100, "dupaccount"),
("ADS_UF_NORMAL_ACCOUNT" , 0x00000200, "useracct"),
("ADS_UF_INTERDOMAIN_TRUST_ACCOUNT" , 0x00000800, "interdomain"),
("ADS_UF_WORKSTATION_TRUST_ACCOUNT" , 0x00001000, "workstation"),
("ADS_UF_SERVER_TRUST_ACCOUNT" , 0x00002000, "server"),
("ADS_UF_DONT_EXPIRE_PASSWD" , 0x00010000, "pwnoexpire"),
("ADS_UF_MNS_LOGON_ACCOUNT" , 0x00020000, "mnslogon"),
("ADS_UF_SMARTCARD_REQUIRED" , 0x00040000, "smartcard"),
("ADS_UF_TRUSTED_FOR_DELEGATION" , 0x00080000, "trustdeleg"),
("ADS_UF_NOT_DELEGATED" , 0x00100000, "notdeleg"),
("ADS_UF_USE_DES_KEY_ONLY" , 0x00200000, "deskey"),
("ADS_UF_DONT_REQUIRE_PREAUTH" , 0x00400000, "nopreauth"),
("ADS_UF_PASSWORD_EXPIRED" , 0x00800000, "pwexpired"),
("ADS_UF_TRUSTED_TO_AUTHENTICATE_FOR_DELEGATION", 0x01000000, "trustauth"),
]
# UAC conversion function
def ConvertUAC(nUAC):
s = ""
for c in lUACCodes:
if ((nUAC & c[1]) == c[1]):
s = s + c[2] + " "
return s
# test routine
if __name__ == "__main__":
print("UACCodes Test")
print("-------------")
for n in [0, 512, 514, 65535]:
print("%d = %s" % (n, ConvertUAC(n)))
print
for s in sys.argv[1:]:
n = int(s)
print("%d = %s" % (n, ConvertUAC(n)))
###
Both modules have some usage examples and should be fairly easy to figure out, but let me know if you have any questions or comments.
There is a Pure Python implementation of an LDAP client called Ldaptor. I don't think it's maintained though. If you really need it, you might be able to run 2to3 on this and port it.
This answer is no longer accurate; see below for other answers.
Sorry to break this on you, but I don't think there is a python-ldap for Python 3 (yet)...
That's the reason why we should keep active development at Python 2.6 for now (as long as most crucial dependencies (libs) are not ported to 3.0).