Develop Reference

gcp pubsub-lite subscription python "AuthMetadataPluginCallback \"<google.auth.transport.grpc.AuthMetadataPlugin object at " raised exception!" error

I am using the python pubsublite client(async version) for subscribing from pubsub-lite.
I get the below error intermittently
Traceback (most recent call last):
File \"/usr/local/lib/python3.10/site-packages/grpc/_plugin_wrapping.py\", line 89, in __call__
self._metadata_plugin(
File \"/usr/local/lib/python3.10/site-packages/google/auth/transport/grpc.py\", line 101, in __call__
callback(self._get_authorization_headers(context), None)
File \"/usr/local/lib/python3.10/site-packages/google/auth/transport/grpc.py\", line 87, in _get_authorization_headers
self._credentials.before_request(
File \"/usr/local/lib/python3.10/site-packages/google/auth/credentials.py\", line 134, in before_request
self.apply(headers)
File \"/usr/local/lib/python3.10/site-packages/google/auth/credentials.py\", line 110, in apply
_helpers.from_bytes(token or self.token)
File \"/usr/local/lib/python3.10/site-packages/google/auth/_helpers.py\", line 130, in from_bytes
raise ValueError(\"{0!r} could not be converted to unicode\".format(value))
ValueError: None could not be converted to unicode"
I don't use GOOGLE_APPLICATION_CREDENTIALS env variable to specify credentials, instead I do as below(I don't want to write credentials to a file in aws host)
import asyncio
from google.cloud.pubsublite.cloudpubsub import AsyncSubscriberClient
from google.cloud.pubsublite.types import (
CloudRegion,
CloudZone,
FlowControlSettings,
SubscriptionPath,
)
from google.oauth2 import service_account
class AsyncTimedIterable:
def __init__(self, iterable, poll_timeout=90):
class AsyncTimedIterator:
def __init__(self):
self._iterator = iterable.__aiter__()
async def __anext__(self):
try:
result = await asyncio.wait_for(
self._iterator.__anext__(), int(poll_timeout)
)
if not result:
raise StopAsyncIteration
return result
except asyncio.TimeoutError as e:
raise e
self._factory = AsyncTimedIterator
def __aiter__(self):
return self._factory()
# TODO add project info below
location = CloudZone(CloudRegion("region"), "zone")
subscription_path = SubscriptionPath("project_number", location, "subscription_id")
# TODO add service account details
gcp_creds = {}
async def async_receive_from_subscription(per_partition_count=100):
# Configure when to pause the message stream for more incoming messages based on the
# maximum size or number of messages that a single-partition subscriber has received,
# whichever condition is met first.
per_partition_flow_control_settings = FlowControlSettings(
# 1,000 outstanding messages. Must be >0.
messages_outstanding=per_partition_count,
# 10 MiB. Must be greater than the allowed size of the largest message (1 MiB).
bytes_outstanding=10 * 1024 * 1024,
)
async with AsyncSubscriberClient(
credentials=service_account.Credentials.from_service_account_info(gcp_creds)
) as async_subscriber_client:
message_iterator = await async_subscriber_client.subscribe(
subscription_path,
per_partition_flow_control_settings=per_partition_flow_control_settings,
)
timed_iter = AsyncTimedIterable(message_iterator, 90)
async for message in timed_iter:
yield message
async def main():
async for message in async_receive_from_subscription(per_partition_count=100_000):
print(message.data)
if __name__ == "__main__":
asyncio.run(main())
when I went through the files in stack trace I saw a code comment as below in file ``
# The plugin may be invoked on a thread created by Core, which will not
# have the context propagated. This context is stored and installed in
# the thread invoking the plugin.
Is it because the credentials I set are not being sent to another thread when it is created?

How to run an infinite loop asynchronously in a websocket with Tornado

I am running a web server on a raspberry pi which is logging temperatures etc.
I am using websockets in Tornado to communicate with my client.
I want the client to be able to control when the server is sending data over the socket.
My thought was that when client connects and says it's ready, the server will start a loop where it logs temps every second. But I need this loop to run asynchronously. This is where I get in trouble. I tried to follow an example, but I cant get it to run properly.
class TemperatureSocketHandler(tornado.websocket.WebSocketHandler):
#gen.coroutine
def async_func(self):
num = 0
while(self.sending):
num = num + 1
temp = self.sense.get_temperature()
yield self.write_message(str(temp))
gen.sleep(1)
def open(self):
print("Temperature socket opened")
self.sense = SenseHat()
self.sense.clear()
self.sending = False
def on_message(self, message):
if(message == "START"):
self.sending = True
if(message == "STOP"):
self.sending = False
tornado.ioloop.IOLoop.current().spawn_callback(self.async_func(self))
But I get an error here when I run this:
ERROR:tornado.application:Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x75159858>)
Traceback (most recent call last):
File "/home/pi/.local/lib/python3.5/site-packages/tornado/ioloop.py", line 605, in _run_callback
ret = callback()
File "/home/pi/.local/lib/python3.5/site-packages/tornado/stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
TypeError: 'Future' object is not callable

You have to use IOLoop.add_future()
since async_func() returns a Future (it is decorated as a coroutine!).
Also, you should add the future, when you receive the start message, not on any message:
def on_message(self, message):
if(message == "START"):
self.sending = True
tornado.ioloop.IOLoop.current().add_future(
self.async_func(self), lambda f: self.close())
if(message == "STOP"):
self.sending = False

Python MQTT to publish multiple message with same topic

I am trying to publish a multiple random data using mqtt to the broker. Below is the script for the publish part.
import paho.mqtt.client as mqtt
import json, schedule, time, random
client = mqtt.Client()
client.connect("<broker address", 1883, 60)
def pub_message():
tempreading = random.uniform(0, 100)
pHreading = random.uniform(1,14)
oxyreading = random.uniform(0, 100)
data_string1 = str(oxyreading)
data_string2 = str(pHreading)
data_string3 = str(tempreading)
msgs = [("randomdata", data_string1),("randomdata", data_string2),("randomdata", data_string3)]
client.publish(msgs)
schedule.every(1).minutes.do(pub_message)
while True:
schedule.run_pending()
time.sleep(1)
client.disconnect()
I ran the script and there is error like below:
Traceback (most recent call last):
File "mqttpub.py", line 27, in <module>
schedule.run_pending()
File "/usr/local/lib/python2.7/dist-packages/schedule/__init__.py", line 462, in run_pending
default_scheduler.run_pending()
File "/usr/local/lib/python2.7/dist-packages/schedule/__init__.py", line 75, in run_pending
self._run_job(job)
File "/usr/local/lib/python2.7/dist-packages/schedule/__init__.py", line 129, in _run_job
ret = job.run()
File "/usr/local/lib/python2.7/dist-packages/schedule/__init__.py", line 377, in run
ret = self.job_func()
File "mqttpub.py", line 22, in pub_message
client.publish(msgs)
File "/usr/local/lib/python2.7/dist-packages/paho/mqtt/client.py", line 980, in publish
rc = self._send_publish(local_mid, topic, local_payload, qos, retain, False, info)
File "/usr/local/lib/python2.7/dist-packages/paho/mqtt/client.py", line 1979, in _send_publish
utopic = topic.encode('utf-8')
AttributeError: 'list' object has no attribute 'encode'
I searched about the publish multiple message with mqtt but did not find any good reference. I also included my mqtt subscribe part for receiving the multiple messages. I did search about this part too but did not find any good reference.
import paho.mqtt.client as mqtt
from models import *
from sqlalchemy.orm import sessionmaker
import json
def on_connect(client, userdata, rc):
print("connected with result code" + str(rc))
client.subscribe("randomdata")
def on_message(client, userdata, msg):
print "Topic:", msg.topic + " " + "Message:" + " " + "Value1:" + str(msg.payload1) + " " + "Value2:" + str(msg.payload2) + " " + "Value3:" + str(msg.payload3)
engine = create_engine('postgresql://user:password#localhost/mydatabase')
Base.metadata.bind = engine
DBSession = sessionmaker(bind=engine)
session = DBSession()
# store message received into database
raw_data = _Data(Value1=msg.payload1, Value2=msg.payload2, Value3=msg.payload3, time=msg.timestamp)
session.add(raw_data)
session.commit()
client = mqtt.Client()
client.on_connect = on_connect
client.on_message = on_message
client.connect("<broker address>",1883, 60)
client.loop_forever()
Does anyone have any experience doing it? Thank you in advance.

What makes you think client.publish() will accept an array?
The doc's (https://pypi.python.org/pypi/paho-mqtt/1.1#publishing) don't mention anything about publishing multiple messages, you will have to call client.publish() once for every message you want to send.
You should also be calling client.loop() in your while true loop.
while True:
schedule.run_pending()
client.loop()
time.sleep(1)

Pyro: cannot execute callbacks

Using Pyro4 I haven't managed to successfully execute a callback from server to client.
The server script looks as follows:
class RobotController(object):
def __init__(self):
self.robotStates = []
def moveDemo(self, motionTime):
stopTime = datetime.datetime.now() + datetime.timedelta(0,motionTime)
while datetime.datetime.now() < stopTime:
print "Robot is moving..."
time.sleep(1)
print "Robot stopped"
return 0
def doCallback(self, callback):
print("server: doing callback 1 to client")
try:
callback.call1()
except:
print("got an exception from the callback.")
print("".join(Pyro4.util.getPyroTraceback()))
print("server: doing callback 2 to client")
try:
callback.call2()
except:
print("got an exception from the callback.")
print("".join(Pyro4.util.getPyroTraceback()))
print("server: callbacks done")
if __name__ == '__main__':
robotController = RobotController()
if os.name == 'posix':
daemon = Pyro4.Daemon(host="192.168.1.104", port=8000);
else:
daemon = Pyro4.Daemon(host="localhost", port=8000);
Pyro4.Daemon.serveSimple(
{ robotController: "robotController"},
ns=False,
daemon=daemon,
verbose = True
)
and the client looks as follows:
class CallbackHandler(object):
def crash(self):
a=1
b=0
return a//b
def call1(self):
print("callback 1 received from server!")
print("going to crash - you won't see the exception here, only on the server")
return self.crash()
#Pyro4.callback
def call2(self):
print("callback 2 received from server!")
print("going to crash - but you will see the exception here too")
return self.crash()
daemon = Pyro4.core.Daemon()
callback = CallbackHandler()
daemon.register(callback)
#robotController = Pyro4.Proxy("PYRO:robotController#192.168.1.104:8000")
robotController = Pyro4.Proxy("PYRO:robotController#localhost:8000")
robotController._pyroOneway.add("doCallback")
robotController.doCallback(callback)
When executing the command robotController.doCallback(callback), the method doCallback on server is executed, but the server cannot access the client. It returns the following:
Traceback (most recent call last):
File "C:\LASTNO\Python Projects\PiBot\RobotServer\PyroServer\pyroServer.py", line 63, in doCallback
callback.call2()
File "C:\Python27\lib\site-packages\Pyro4\core.py", line 160, in __call__
return self.__send(self.__name, args, kwargs)
File "C:\Python27\lib\site-packages\Pyro4\core.py", line 286, in _pyroInvoke
self.__pyroCreateConnection()
File "C:\Python27\lib\site-packages\Pyro4\core.py", line 371, in __pyroCreateConnection
raise ce
CommunicationError: cannot connect: [Errno 10061] No connection could be made because the target machine actively refused it
Does anyone know what could be the cause of the error and how to fix it? Thank you!

I solved the problem by modifying the client code as follows:
class CallbackHandler(object):
def crash(self):
a=1
b=0
return a//b
def call1(self):
print("callback 1 received from server!")
print("going to crash - you won't see the exception here, only on the server")
return self.crash()
#Pyro4.callback
def call2(self):
print("callback 2 received from server!")
print("going to crash - but you will see the exception here too")
return self.crash()
daemon = Pyro4.core.Daemon()
callback = CallbackHandler()
daemon.register(callback)
with Pyro4.core.Proxy("PYRO:robotController#localhost:8000") as server:
server._pyroOneway.add("doCallback")
server.doCallback(callback)
motion = server.moveDemo(15)
print("waiting for callbacks to arrive...")
print("(ctrl-c/break the program once it's done)\n")
daemon.requestLoop()

Ping and Pong in a threaded WebSocket server (Python)

I've written a threaded websocket server in Python, using the lastest websocket spec and I'm trying to make it send a ping request to every client every x seconds. The only way I came up with to do this is overriding BaseServer.server_forever() like this:
# override BaseServer's serve_forever to send a ping request every now and then
class ModTCPServer(SocketServer.TCPServer):
def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True):
SocketServer.TCPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate)
self.__is_shut_down = threading.Event()
self.__shutdown_request = False
def serve_forever(self, poll_interval=0.5):
###
global PING_EVERY_SEC
self.lastPing = int(time())
###
self.__is_shut_down.clear()
try:
while not self.__shutdown_request:
r, w, e = select.select([self], [], [], poll_interval)
if self in r:
self._handle_request_noblock()
###
now = int(time())
if (now - self.lastPing) >= PING_EVERY_SEC:
self.socket.send(WebSocketPing(['0x89','0x21','0xa7','0x4b'], now)) # arbitrary key
self.lastPing = now
###
finally:
self.__shutdown_request = False
self.__is_shut_down.set()
class LoginServer(SocketServer.ThreadingMixIn, ModTCPServer):
pass
server = LoginServer(("", PORT), ApplicationHandler)
print "serving at port", PORT
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
while server_thread.isAlive():
pass
server.shutdown()
Here is the function that constructs the Ping frame, it just puts the timestamp in the contents:
def WebSocketPing(key, timestamp=False):
data = ['0x89','0x8a'] # 0x89 = fin,ping 0x8a = masked,len=10
data.extend(key)
if timestamp:
t = str(timestamp)
else:
t = str(int(time()))
for i in range(10):
masking_byte = int(key[i%4],16)
masked = ord(t[i])
data.append(hex(masked ^ masking_byte))
frame = ''
for i in range(len(data)):
frame += chr(int(data[i],16))
return frame
Bad things happen when I run this
Traceback (most recent call last):
File "LoginServer.py", line 91, in <module>
server = LoginServer(("", PORT), ApplicationHandler)
File "LoginServer.py", line 63, in __init__
SocketServer.TCPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate)
File "/usr/lib/python2.6/SocketServer.py", line 400, in __init__
self.server_bind()
File "/usr/lib/python2.6/SocketServer.py", line 411, in server_bind
self.socket.bind(self.server_address)
File "<string>", line 1, in bind
socket.error: [Errno 112] Address already in use
I assume this is down to my lack of understanding of how overriding works in Python or to a fundamentally wrong approach to this problem. Is there a better way to do this or a way to make this code work?

The codes does not set the properties __is_shut_down and __shutdown_request anywhere. Therefore, trying to access them fails. Create them in the constructor, like this:
class ModTCPServer(SocketServer.TCPServer):
def __init__(self, *args, **kwargs):
SocketServer.TCPServer.__init__(self, *args, **kwargs)
self.__is_shut_down = threading.Event()
self.__shutdown_request = threading.Event()
In response to the update:
socket.error: [Errno 112] Address already in use
means that another process has already bound to the specified port. On Unix, you can find that process with sudo netstat -ltpn. Alternatively, choose a different port.