I,m just study codding, and try make chat-bot. But it's don' work, please help me.
I'ts my code:
import time
import logging
from aiogram import Bot, Dispatcher, types, executor
logging.basicConfig(level=logging.INFO)
TOKEN = '6292675773:AAGBBeUms_4NILy6h26TH4rGLyml7h3bHWg'
MSG = 'Hello, are you coddind today?'
bot = Bot(token=TOKEN)
dp = Dispatcher(bot=bot)
#dp.message_handler(Commands=['start'])
async def start_handler(message: types.Message):
user_id = message.from_user.id
user_name = message.from_user.first_name
user_full_name = message.from_user.first_name
logging.info(f'{user_id} {user_full_name} {time.asctime()}')
await message.reply(f"Hello, {user_full_name}!")
for i in range(10):
time.sleep(2)
await bot.send_message(user_id, MSG.format(user_name))
if __name__ == '__main__':
executor.start_polling(dp)
and it's Error:
(venv) PS C:\Users\roman\PycharmProjects\pythonProject> python main.py
Traceback (most recent call last):
File "C:\Users\roman\PycharmProjects\pythonProject\main.py", line 14, in
#dp.message_handler(Commands = ['start'])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\roman\PycharmProjects\pythonProject\venv\Lib\site-packages\aiogram\dispatcher\dispatcher.py", line 560, in decorator
self.register_message_handler(callback, *custom_filters,
File "C:\Users\roman\PycharmProjects\pythonProject\venv\Lib\site-packages\aiogram\dispatcher\dispatcher.py", line 479, in register_message_handler
filters_set = self.filters_factory.resolve(self.message_handlers,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\roman\PycharmProjects\pythonProject\venv\Lib\site-packages\aiogram\dispatcher\filters\factory.py", line 51, in resolve
filters_set = list(
^^^^^
File "C:\Users\roman\PycharmProjects\pythonProject\venv\Lib\site-packages\aiogram\dispatcher\filters\factory.py", line 77, in _resolve_registered
raise NameError("Invalid filter name(s): '" + "', ".join(full_config.keys()) + "'")
NameError: Invalid filter name(s): 'Commands'
I am using the python pubsublite client(async version) for subscribing from pubsub-lite.
I get the below error intermittently
Traceback (most recent call last):
File \"/usr/local/lib/python3.10/site-packages/grpc/_plugin_wrapping.py\", line 89, in __call__
self._metadata_plugin(
File \"/usr/local/lib/python3.10/site-packages/google/auth/transport/grpc.py\", line 101, in __call__
callback(self._get_authorization_headers(context), None)
File \"/usr/local/lib/python3.10/site-packages/google/auth/transport/grpc.py\", line 87, in _get_authorization_headers
self._credentials.before_request(
File \"/usr/local/lib/python3.10/site-packages/google/auth/credentials.py\", line 134, in before_request
self.apply(headers)
File \"/usr/local/lib/python3.10/site-packages/google/auth/credentials.py\", line 110, in apply
_helpers.from_bytes(token or self.token)
File \"/usr/local/lib/python3.10/site-packages/google/auth/_helpers.py\", line 130, in from_bytes
raise ValueError(\"{0!r} could not be converted to unicode\".format(value))
ValueError: None could not be converted to unicode"
I don't use GOOGLE_APPLICATION_CREDENTIALS env variable to specify credentials, instead I do as below(I don't want to write credentials to a file in aws host)
import asyncio
from google.cloud.pubsublite.cloudpubsub import AsyncSubscriberClient
from google.cloud.pubsublite.types import (
CloudRegion,
CloudZone,
FlowControlSettings,
SubscriptionPath,
)
from google.oauth2 import service_account
class AsyncTimedIterable:
def __init__(self, iterable, poll_timeout=90):
class AsyncTimedIterator:
def __init__(self):
self._iterator = iterable.__aiter__()
async def __anext__(self):
try:
result = await asyncio.wait_for(
self._iterator.__anext__(), int(poll_timeout)
)
if not result:
raise StopAsyncIteration
return result
except asyncio.TimeoutError as e:
raise e
self._factory = AsyncTimedIterator
def __aiter__(self):
return self._factory()
# TODO add project info below
location = CloudZone(CloudRegion("region"), "zone")
subscription_path = SubscriptionPath("project_number", location, "subscription_id")
# TODO add service account details
gcp_creds = {}
async def async_receive_from_subscription(per_partition_count=100):
# Configure when to pause the message stream for more incoming messages based on the
# maximum size or number of messages that a single-partition subscriber has received,
# whichever condition is met first.
per_partition_flow_control_settings = FlowControlSettings(
# 1,000 outstanding messages. Must be >0.
messages_outstanding=per_partition_count,
# 10 MiB. Must be greater than the allowed size of the largest message (1 MiB).
bytes_outstanding=10 * 1024 * 1024,
)
async with AsyncSubscriberClient(
credentials=service_account.Credentials.from_service_account_info(gcp_creds)
) as async_subscriber_client:
message_iterator = await async_subscriber_client.subscribe(
subscription_path,
per_partition_flow_control_settings=per_partition_flow_control_settings,
)
timed_iter = AsyncTimedIterable(message_iterator, 90)
async for message in timed_iter:
yield message
async def main():
async for message in async_receive_from_subscription(per_partition_count=100_000):
print(message.data)
if __name__ == "__main__":
asyncio.run(main())
when I went through the files in stack trace I saw a code comment as below in file ``
# The plugin may be invoked on a thread created by Core, which will not
# have the context propagated. This context is stored and installed in
# the thread invoking the plugin.
Is it because the credentials I set are not being sent to another thread when it is created?
I have a pyglet program that runs a GUI in one thread and receives a message from another thread. It's supposed to update the GUI based on that message. The problem though is whenever I go to update a GUI element, I get an error message like this one:
Traceback (most recent call last):
File --- line 40, in run
pub.sendMessage("update", msg="Banana")
File --- line 77, in updateDisplay
self.label.text = msg
File "---\pyglet\text\__init__.py", line 289, in text
self.document.text = text
File "---\pyglet\text\document.py", line 294, in text
self.insert_text(0, text)
File "---\pyglet\text\document.py", line 425, in insert_text
self.dispatch_event('on_insert_text', start, text)
File "---\pyglet\event.py", line 408, in dispatch_event
if handler(*args):
File "---\pyglet\text\layout.py", line 1045, in on_insert_text
self._init_document()
File "---\pyglet\text\layout.py", line 1034, in _init_document
self._update()
File "---\pyglet\text\layout.py", line 957, in _update
lines = self._get_lines()
File "---\pyglet\text\layout.py", line 933, in _get_lines
glyphs = self._get_glyphs()
...
File "---\pyglet\font\base.py", line 246, in fit
region.blit_into(image, 0, 0, 0)
File "---\pyglet\image\__init__.py", line 1730, in blit_into
self.owner.blit_into(source, x + self.x, y + self.y, z + self.z)
File "---\pyglet\image\__init__.py", line 1622, in blit_into
glBindTexture(self.target, self.id)
File "---\pyglet\gl\lib.py", line 107, in errcheck
raise GLException(msg)
pyglet.gl.lib.GLException: b'invalid operation'
The operative error being pyglet.gl.lib.GLException: b'invalid operation'
I know self.label.text = Y works because it updates the text if I put it in the init() function. However, trying to do any kind of update from the update_display function (see code below) throws this error. I know the function gets called correctly because it successfully prints the message to console, but it breaks on the update element line. I've tried other elements than label and it's the same thing.
I have no idea why it's throwing this error or how to resolve it. Any ideas?
My code is:
import pyglet
from threading import Thread
from random import choice
from time import sleep
from pubsub import pub
RUN = True
class MessageThread(Thread):
def __init__(self):
"""Init Thread Class."""
Thread.__init__(self)
self.start() # start the thread
def run(self):
"""Run Thread."""
array = [0, 1, 2]
while True:
if RUN == False:
print("End runner")
break
value = choice(array)
sleep(1)
if value == 0:
print("event", value, ": Apple", flush=True)
pub.sendMessage("update", msg="Apple")
elif value == 1:
print("event", value, ": Banana", flush=True)
pub.sendMessage("update", msg="Banana")
else:
print("event", value, ": Carrot", flush=True)
pub.sendMessage("update", msg="Carrot")
class MyGUI(pyglet.window.Window):
# Note: With this structure, there is no "self.app", you just use
# self. This structure allows onDraw() to be called.
def __init__(self):
super(MyGUI, self).__init__(width=600,height=650,resizable=True)
self.label = pyglet.text.Label('Waiting...',
font_name='Times New Roman',
font_size=36,
x=self.width/2, y=self.height - 36,
anchor_x='center', anchor_y='center')
# create a pubsub receiver
# update is the topic subscribed to, updateDisplay the callable
pub.subscribe(self.updateDisplay, "update")
MessageThread()
def on_draw(self):
self.clear()
self.label.draw()
def updateDisplay(self, msg):
""" Receives data from thread and updates the display """
print ("Updating Display")
self.label.text = msg
def on_close(self):
""" Override the normal onClose behavior so that we can
kill the MessageThread, too
"""
print("Closing Application")
global RUN
RUN = False
self.close() # Need to include since defining on_close overrides it
if __name__ == "__main__":
window = MyGUI()
pyglet.app.run()
I am trying to make a makeblock mblock implementation with python and i found this api on github but it uses python 2.7 and i am using python 3.8 so i am trying to configure the code so it can run on python 3.8 but i ran into an error which is
init mBot
bot
<lib.mBot.mSerial object at 0x02CE9898>
start with serial
<lib.mBot.mHID object at 0x02E2B310>
self.device = mHID()
'mHID' object has no attribute 'dict'
Exception in thread Thread-2:
self.start()
Traceback (most recent call last):
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 154, in __onRead
start with HID
--------------------
n = self.device.inWaiting()
<lib.mBot.mBot object at 0x002FF640>
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 96, in inWaiting
--------------------
Error in sys.excepthook:
buf = self.dict.device.read(64)
Traceback (most recent call last):
AttributeError: 'mHID' object has no attribute 'dict'
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 134, in excepthook
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\abdsak11\AppData\Local\Programs\Python\Python38-32\lib\threading.py", line 932, in _bootstrap_inner
self.close()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 142, in close
self.device.close()
self.run()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 106, in close
File "C:\Users\abdsak11\AppData\Local\Programs\Python\Python38-32\lib\threading.py", line 870, in run
self.dict.device.close()
AttributeError: 'mHID' object has no attribute 'dict'
Original exception was:
Traceback (most recent call last):
File "c:/Users/abdsak11/OneDrive - Lärande/Dokument/GitHub/python-for-mbot/test.py", line 15, in <module>
self._target(*self._args, **self._kwargs)
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 163, in __onRead
bot.doMove(50,50)
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 179, in doMove
self.close()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 142, in close
self.__writePackage(bytearray([0xff,0x55,0x7,0x0,0x2,0x5]+self.short2bytes(-leftSpeed)+self.short2bytes(rightSpeed)))
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 278, in short2bytes
self.device.close()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 106, in close
return [ord(val[0]),ord(val[1])]
TypeError: ord() expected string of length 1, but int found
self.dict.device.close()
AttributeError: 'mHID' object has no attribute 'dict'
'mHID' object has no attribute 'dict'
Exception in thread Thread-1:
Traceback (most recent call last):
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 154, in __onRead
n = self.device.inWaiting()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 96, in inWaiting
buf = self.dict.device.read(64)
AttributeError: 'mHID' object has no attribute 'dict'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\abdsak11\AppData\Local\Programs\Python\Python38-32\lib\threading.py", line 932, in _bootstrap_inner
self.run()
File "C:\Users\abdsak11\AppData\Local\Programs\Python\Python38-32\lib\threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 163, in __onRead
self.close()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 142, in close
self.device.close()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 106, in close
self.dict.device.close()
AttributeError: 'mHID' object has no attribute 'dict'
So what i am trying to do is to control and upload code to my mbot via usb COM3 serial and thats the error that i am getting i tried everthing but its not working and i cant seem to figure out the cause of the error.
mbot.py
this is the main api that i found
import serial
import sys,time
import signal
from time import ctime,sleep
import glob,struct
from multiprocessing import Process,Manager,Array
import threading
import hid
class mSerial():
ser = None
def __init__(self):
print (self)
def start(self, port):
self.ser = serial.Serial(port,115200)
def device(self):
return self.ser
def serialPorts(self):
if sys.platform.startswith('win'):
ports = ['COM%s' % (i + 1) for i in range(256)]
elif sys.platform.startswith('linux') or sys.platform.startswith('cygwin'):
ports = glob.glob('/dev/tty[A-Za-z]*')
elif sys.platform.startswith('darwin'):
ports = glob.glob('/dev/tty.*')
else:
raise EnvironmentError('Unsupported platform')
result = []
for port in ports:
s = serial.Serial()
s.port = port
s.close()
result.append(port)
return result
def writePackage(self,package):
self.ser.write(package)
sleep(0.01)
def read(self):
return self.ser.read()
def isOpen(self):
return self.ser.isOpen()
def inWaiting(self):
return self.ser.inWaiting()
def close(self):
self.ser.close()
class mHID():
def __init__(self):
print (self)
def start(self):
print ("starting start \n\n")
self.manager = Manager()
print("manager pass \n\n")
self.dict = self.manager.dict()
print("dict pass \n\n")
self.dict.device = hid.device()
print("dict device pass \n\n")
self.dict.device.open(0x0416, 0xffff)
print("dict device open pass \n\n")
self.dict.device.hid_set_nonblocking(self.device,1)
print ("start")
self.buffer = []
self.bufferIndex = 0
def enumerate(self):
print ("enumerate")
for dev in self.dict.device.enumerate():
print ('------------------------------------------------------------')
print (dev.description())
def writePackage(self,package):
buf = []
buf += [0, len(package)]
for i in range(len(package)):
buf += [package[i]]
n = self.dict.device.write(buf)
sleep(0.01)
def read(self):
c = self.buffer[0]
self.buffer = self.buffer[1:]
return chr(c)
def isOpen(self):
return True
def inWaiting(self):
buf = self.dict.device.read(64)
l = 0
if len(buf)>0:
l = buf[0]
if l>0:
for i in range(0,l):
self.buffer += [buf[i+1]]
return len(self.buffer)
def close(self):
self.dict.device.close()
class mBot():
def __init__(self):
print ("init mBot")
signal.signal(signal.SIGINT, self.exit)
self.manager = Manager()
self.__selectors = self.manager.dict()
self.buffer = []
self.bufferIndex = 0
self.isParseStart = False
self.exiting = False
self.isParseStartIndex = 0
def startWithSerial(self, port):
self.device = mSerial()
self.device.start(port)
self.start()
def startWithHID(self):
self.device = mHID()
print ("self.device = mHID()\n \n")
self.device.start()
print ("self.device.start()\n \n")
self.start()
print ("self.start()\n \n")
def excepthook(self, exctype, value, traceback):
self.close()
def start(self):
sys.excepthook = self.excepthook
th = threading.Thread(target=self.__onRead,args=(self.onParse,))
th.start()
def close(self):
self.device.close()
def exit(self, signal, frame):
self.exiting = True
sys.exit(0)
def __onRead(self,callback):
while 1:
if(self.exiting==True):
break
try:
if self.device.isOpen()==True:
n = self.device.inWaiting()
for i in range(n):
r = ord(self.device.read())
callback(r)
sleep(0.01)
else:
sleep(0.5)
except Exception as ex:
print (str(ex))
self.close()
sleep(1)
def __writePackage(self,pack):
self.device.writePackage(pack)
def doRGBLed(self,port,slot,index,red,green,blue):
self.__writePackage(bytearray([0xff,0x55,0x9,0x0,0x2,0x8,port,slot,index,red,green,blue]))
def doRGBLedOnBoard(self,index,red,green,blue):
self.doRGBLed(0x7,0x2,index,red,green,blue)
def doMotor(self,port,speed):
self.__writePackage(bytearray([0xff,0x55,0x6,0x0,0x2,0xa,port]+self.short2bytes(speed)))
def doMove(self,leftSpeed,rightSpeed):
self.__writePackage(bytearray([0xff,0x55,0x7,0x0,0x2,0x5]+self.short2bytes(-leftSpeed)+self.short2bytes(rightSpeed)))
def doServo(self,port,slot,angle):
self.__writePackage(bytearray([0xff,0x55,0x6,0x0,0x2,0xb,port,slot,angle]))
def doBuzzer(self,buzzer,time=0):
self.__writePackage(bytearray([0xff,0x55,0x7,0x0,0x2,0x22]+self.short2bytes(buzzer)+self.short2bytes(time)))
def doSevSegDisplay(self,port,display):
self.__writePackage(bytearray([0xff,0x55,0x8,0x0,0x2,0x9,port]+self.float2bytes(display)))
def doIROnBoard(self,message):
self.__writePackage(bytearray([0xff,0x55,len(message)+3,0x0,0x2,0xd,message]))
def requestLightOnBoard(self,extID,callback):
self.requestLight(extID,8,callback)
def requestLight(self,extID,port,callback):
self.__doCallback(extID,callback)
self.__writePackage(bytearray([0xff,0x55,0x4,extID,0x1,0x3,port]))
def requestButtonOnBoard(self,extID,callback):
self.__doCallback(extID,callback)
self.__writePackage(bytearray([0xff,0x55,0x4,extID,0x1,0x1f,0x7]))
def requestIROnBoard(self,extID,callback):
self.__doCallback(extID,callback)
self.__writePackage(bytearray([0xff,0x55,0x3,extID,0x1,0xd]))
def requestUltrasonicSensor(self,extID,port,callback):
self.__doCallback(extID,callback)
self.__writePackage(bytearray([0xff,0x55,0x4,extID,0x1,0x1,port]))
def requestLineFollower(self,extID,port,callback):
self.__doCallback(extID,callback)
self.__writePackage(bytearray([0xff,0x55,0x4,extID,0x1,0x11,port]))
def onParse(self, byte):
position = 0
value = 0
self.buffer+=[byte]
bufferLength = len(self.buffer)
if bufferLength >= 2:
if (self.buffer[bufferLength-1]==0x55 and self.buffer[bufferLength-2]==0xff):
self.isParseStart = True
self.isParseStartIndex = bufferLength-2
if (self.buffer[bufferLength-1]==0xa and self.buffer[bufferLength-2]==0xd and self.isParseStart==True):
self.isParseStart = False
position = self.isParseStartIndex+2
extID = self.buffer[position]
position+=1
type = self.buffer[position]
position+=1
# 1 byte 2 float 3 short 4 len+string 5 double
if type == 1:
value = self.buffer[position]
if type == 2:
value = self.readFloat(position)
if(value<-255 or value>1023):
value = 0
if type == 3:
value = self.readShort(position)
if type == 4:
value = self.readString(position)
if type == 5:
value = self.readDouble(position)
if(type<=5):
self.responseValue(extID,value)
self.buffer = []
def readFloat(self, position):
v = [self.buffer[position], self.buffer[position+1],self.buffer[position+2],self.buffer[position+3]]
return struct.unpack('<f', struct.pack('4B', *v))[0]
def readShort(self, position):
v = [self.buffer[position], self.buffer[position+1]]
return struct.unpack('<h', struct.pack('2B', *v))[0]
def readString(self, position):
l = self.buffer[position]
position+=1
s = ""
for i in Range(l):
s += self.buffer[position+i].charAt(0)
return s
def readDouble(self, position):
v = [self.buffer[position], self.buffer[position+1],self.buffer[position+2],self.buffer[position+3]]
return struct.unpack('<f', struct.pack('4B', *v))[0]
def responseValue(self, extID, value):
self.__selectors["callback_"+str(extID)](value)
def __doCallback(self, extID, callback):
self.__selectors["callback_"+str(extID)] = callback
def float2bytes(self,fval):
val = struct.pack("f",fval)
return [ord(val[0]),ord(val[1]),ord(val[2]),ord(val[3])]
def short2bytes(self,sval):
val = struct.pack("h",sval)
return [ord(val[0]),ord(val[1])]
test.py
this is the test file that i am working with
from lib.mBot import *
if __name__ == "__main__":
bot = mBot()
print ("bot \n\n")
bot.startWithSerial("COM3")
print ("start with serial\n\n")
bot.startWithHID()
print ("start with HID \n\n")
print(f'--------------------\n')
print (bot)
print(f'--------------------\n')
bot.doMove(50,50)
I believe your trouble is here:
class mHID():
def __init__(self):
print (self)
def start(self):
# ...
self.dict = self.manager.dict()
# ...
def close(self):
self.dict.device.close()
You should always define your instance attributes in __init__(), at least with an initial value of None, even if you plan to change them later.
Additionally, whenever you have a situation where "A must be called before B", you should write the code to properly handle when B is called without A. In this case, what should close() do if start() was never called? For simplicity, I'm just having it do nothing, but you may consider raising an exception.
class mHID():
def __init__(self):
print (self)
self.dct = None
def start(self):
# ...
self.dct = self.manager.dict()
# ...
def close(self):
if self.dct:
self.dct.device.close()
You'll notice I also renamed self.dict to self.dct. Although the former name may technically be allowable because of the self. namespace, dict is actually a built-in class. Under various circumstances, you could wind up dealing with shadowing. In the very least, it's confusing to always have to remind oneself that "this isn't the built-in dict, but an instance variable instead.
(Possible aside: is there a particular reason why you don't want to call start() from the initializer?)
WARNING: This is not a simple case of "copy and paste the answer". I've illustrated a couple of design principles here that you should incorporate into the rest of your code.
I'm writing multi-process code, which runs perfectly in Python 3.7. Yet I want one of the parallel process to execute an IO process take stakes for ever using AsyncIO i order to get better performance, but have not been able to get it to run.
Ubuntu 18.04, Python 3.7, AsyncIO, pipenv (all pip libraries installed)
The method in particular runs as expected using multithreading, which is what I want to replace with AsyncIO.
I have googled and tried looping in the main() function and now only in the intended cor-routine, have looked at examples and read about this new Async way of getting things down and no results so far.
The following is the app.py code which is esecuted: python app.py
import sys
import traceback
import logging
import asyncio
from config import DEBUG
from config import log_config
from <some-module> import <some-class>
if DEBUG:
logging.config.dictConfig(log_config())
else:
logging.basicConfig(
level=logging.DEBUG, format='%(relativeCreated)6d %(threadName)s %(message)s')
logger = logging.getLogger(__name__)
def main():
try:
<some> = <some-class>([
'some-data1.csv',
'some-data2.csv'
])
<some>.run()
except:
traceback.print_exc()
pdb.post_mortem()
sys.exit(0)
if __name__ == '__main__':
asyncio.run(main())
Here is the code where I have the given class defined
_sql_client = SQLServer()
_blob_client = BlockBlobStore()
_keys = KeyVault()
_data_source = _keys.fetch('some-data')
# Multiprocessing
_manager = mp.Manager()
_ns = _manager.Namespace()
def __init__(self, list_of_collateral_files: list) -> None:
#timeit
def _get_filter_collateral(self, ns: mp.managers.NamespaceProxy) -> None:
#timeit
def _get_hours(self, ns: mp.managers.NamespaceProxy) -> None:
#timeit
def _load_original_bids(self, ns: mp.managers.NamespaceProxy) -> None:
#timeit
def _merge_bids_with_hours(self, ns: mp.managers.NamespaceProxy) -> None:
#timeit
def _get_collaterial_per_month(self, ns: mp.managers.NamespaceProxy) -> None:
#timeit
def _calc_bid_per_path(self) -> None:
#timeit
def run(self) -> None:
The method containing the async code is here:
def _get_filter_collateral(self, ns: mp.managers.NamespaceProxy) -> None:
all_files = self._blob_client.download_blobs(self._list_of_blob_files)
_all_dfs = pd.DataFrame()
async def read_task(file_: str) -> None:
nonlocal _all_dfs
df = pd.read_csv(StringIO(file_.content))
_all_dfs = _all_dfs.append(df, sort=False)
tasks = []
loop = asyncio.new_event_loop()
for file_ in all_files:
tasks.append(asyncio.create_task(read_task(file_)))
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
_all_dfs['TOU'] = _all_dfs['TOU'].map(lambda x: 'OFFPEAK' if x == 'OFF' else 'ONPEAK')
ns.dfs = _all_dfs
And the method that calls the particular sequence and and this async method is:
def run(self) -> None:
extract = []
extract.append(mp.Process(target=self._get_filter_collateral, args=(self._ns, )))
extract.append(mp.Process(target=self._get_hours, args=(self._ns, )))
extract.append(mp.Process(target=self._load_original_bids, args=(self._ns, )))
# Start the parallel processes
for process in extract:
process.start()
# Await for database process to end
extract[1].join()
extract[2].join()
# Merge both database results
self._merge_bids_with_hours(self._ns)
extract[0].join()
self._get_collaterial_per_month(self._ns)
self._calc_bid_per_path()
self._save_reports()
self._upload_data()
These are the errors I get:
Process Process-2:
Traceback (most recent call last):
File "<some-path>/.pyenv/versions/3.7.4/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
self.run()
File "<some-path>/.pyenv/versions/3.7.4/lib/python3.7/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "<some-path>/src/azure/application/utils/lib.py", line 10, in timed
result = method(*args, **kwargs)
File "<some-path>/src/azure/application/caiso/main.py", line 104, in _get_filter_collateral
tasks.append(asyncio.create_task(read_task(file_)))
File "<some-path>/.pyenv/versions/3.7.4/lib/python3.7/asyncio/tasks.py", line 350, in create_task
loop = events.get_running_loop()
RuntimeError: no running event loop
<some-path>/.pyenv/versions/3.7.4/lib/python3.7/multiprocessing/process.py:313: RuntimeWarning: coroutine '<some-class>._get_filter_collateral.<locals>.read_task' was never awaited
traceback.print_exc()
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
DEBUG Calculating monthly collateral...
Traceback (most recent call last):
File "app.py", line 25, in main
caiso.run()
File "<some-path>/src/azure/application/utils/lib.py", line 10, in timed
result = method(*args, **kwargs)
File "<some-path>/src/azure/application/caiso/main.py", line 425, in run
self._get_collaterial_per_month(self._ns)
File "<some-path>/src/azure/application/utils/lib.py", line 10, in timed
result = method(*args, **kwargs)
File "<some-path>/src/azure/application/caiso/main.py", line 196, in _get_collaterial_per_month
credit_margin = ns.dfs
File "<some-path>/.pyenv/versions/3.7.4/lib/python3.7/multiprocessing/managers.py", line 1122, in __getattr__
return callmethod('__getattribute__', (key,))
File "<some-path>/.pyenv/versions/3.7.4/lib/python3.7/multiprocessing/managers.py", line 834, in _callmethod
raise convert_to_error(kind, result)
AttributeError: 'Namespace' object has no attribute 'dfs'
> <some-path>/.pyenv/versions/3.7.4/lib/python3.7/multiprocessing/managers.py(834)_callmethod()
-> raise convert_to_error(kind, result)
(Pdb)
As it seems from the Traceback log it is look like you are trying to add tasks to not running event loop.
/.pyenv/versions/3.7.4/lib/python3.7/multiprocessing/process.py:313:
RuntimeWarning: coroutine
'._get_filter_collateral..read_task' was never
awaited
The loop was just created and it's not running yet, therefor the asyncio unable to attach tasks to it.
The following example will reproduce the same results, adding tasks and then trying to await for all of them to finish:
import asyncio
async def func(num):
print('My name is func {0}...'.format(num))
loop = asyncio.get_event_loop()
tasks = list()
for i in range(5):
tasks.append(asyncio.create_task(func(i)))
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
Results with:
Traceback (most recent call last):
File "C:/tmp/stack_overflow.py", line 42, in <module>
tasks.append(asyncio.create_task(func(i)))
File "C:\Users\Amiram\AppData\Local\Programs\Python\Python37-32\lib\asyncio\tasks.py", line 324, in create_task
loop = events.get_running_loop()
RuntimeError: no running event loop
sys:1: RuntimeWarning: coroutine 'func' was never awaited
Nonetheless the solution is pretty simple, you just need to add the tasks to the created loop - instead of asking the asyncio to go it.
The only change is needed in the following line:
tasks.append(asyncio.create_task(func(i)))
Change the creation of the task from the asyncio to the newly created loop, you are able to do it because this is your loop unlike the asynio which is searching for a running one.
So the new line should look like this:
tasks.append(loop.create_task(func(i)))
Another solution could be running an async function and create the tasks there for example (Because that loop is already running now the asyncio enable to attach tasks to it):
import asyncio
async def func(num):
print('Starting func {0}...'.format(num))
await asyncio.sleep(0.1)
print('Ending func {0}...'.format(num))
loop = asyncio.get_event_loop()
async def create_tasks_func():
tasks = list()
for i in range(5):
tasks.append(asyncio.create_task(func(i)))
await asyncio.wait(tasks)
loop.run_until_complete(create_tasks_func())
loop.close()
This simple change will results with:
Starting func 0...
Starting func 1...
Starting func 2...
Starting func 3...
Starting func 4...
Ending func 0...
Ending func 2...
Ending func 4...
Ending func 1...
Ending func 3...
Use asyncio.ensure_future instead. See https://docs.python.org/3/library/asyncio-future.html#asyncio.ensure_future