Develop Reference

Telegram client operating on multiple thread

I'm trying to attach to Telegram APIs to interact through such platform.
I need to send messages on multiple different threads, but I've not found any solution so far allowing me to do that.
Here you can find two sketch codes of my attempts to manage messages on different threads, using both telethon and pyrogram libraries.
So far, I've not found a solution yet.
Telethon case
import time
import threading
from telethon import TelegramClient, events
telegram_client = None
async def telethon_telegram_init(api_id, api_hash):
global telegram_client
client = TelegramClient('my_account', api_id, api_hash)
await client.start()
try: assert await client.connect()
except: pass
if not await client.is_user_authorized():
client.send_code_request(phone_number)
me = client.sign_in(phone_number, input('Enter code: '))
client.parse_mode = 'html'
telegram_client = client
#telegram_client.on(events.NewMessage(incoming=True))
async def telethon_telegram_receive(event):
if event.is_private: await event.respond('Thank you for your message')
def telethon_telegram_send(to, message):
global telegram_client
while True:
telegram_client.send_message(to, message)
time.sleep(60)
# initiating the client object
telegram_client = telethon_telegram_init(api_id, api_hash)
# running the thread aimed to send messages
threading.Thread(target=telethon_telegram_send, args=('me', 'Hello')).start()
Results:
initialization: ok
message reception: not working (no errors are returned)
message sending: not working (AttributeError: 'coroutine' object has no attribute 'send_message' returned)
Pyrogram case
import time
import threading
from pyrogram import Client, filters
def pyrogram_telegram_init(api_id, api_hash):
client = Client('my_account', api_id, api_hash)
#client.on_message(filters.private)
async def pyrogram_telegram_receive(event):
await event.reply_text(f'Thank you for your message')
client.start()
return client
def pyrogram_telegram_send(client, to, message):
while True:
client.send_message(to, message)
time.sleep(60)
# initiating the client object
telegram_client = pyrogram_telegram_init(api_id, api_hash)
# running the thread aimed to send messages
threading.Thread(target=pyrogram_telegram_send, args=(telegram_client, 'me', 'Hello')).start()
Results:
initialization: ok
message reception: not working (no errors are returned)
message sending: not working (no errors are returned)
Update:
Attempts with asyncio
I've also tried to use asyncio, as suggested by #Lonami. I've focused on the telethon library.
Nevertheless, please note that in my case, new Telegram messages are generated by a TCP server listening on the host, in a way similar to the simplified sketch code below.
Here is the code reporting my attempts.
import time
import asyncio
from telethon import TelegramClient, events
RECIPIENT = 'me'
api_id = '...'
api_hash = '...'
telegram_client = None
# server management part
from socketserver import ThreadingMixIn, TCPServer, StreamRequestHandler
class ThreadingTCPServer(ThreadingMixIn, TCPServer): pass
class ServerSample(StreamRequestHandler):
def handle(self):
message = 'New connection from %s:%s' % self.client_address
print(message)
self.server.loop.call_soon_threadsafe(self.server.queue.put_nowait, message)
# also tried with the following:
#self.server.queue.put_nowait(message)
async def initialize_server(loop, queue):
with ThreadingTCPServer(('127.0.0.1', 8080), ServerSample) as server:
server.loop = loop
server.queue = queue
server.serve_forever()
async def telethon_telegram_init(api_id, api_hash):
global telegram_client
client = TelegramClient('my_account', api_id, api_hash)
await client.start()
try: assert await client.connect()
except Exception as e: print(str(e))
if not await client.is_user_authorized():
client.send_code_request(phone_number)
me = client.sign_in(phone_number, input('Enter code: '))
client.parse_mode = 'html'
telegram_client = client
#telegram_client.on(events.NewMessage(incoming=True))
async def telethon_telegram_receive(event):
if event.is_private: await event.respond('Thank you for your message')
return telegram_client
async def telethon_telegram_generate(loop, queue, t):
loop.call_soon_threadsafe(queue.put_nowait, 'Hello from direct {} call'.format(t))
async def main():
global telegram_client
loop = asyncio.get_running_loop()
queue = asyncio.Queue()
# initiating the Telegram client object
await telethon_telegram_init(api_id, api_hash)
# initializing the server (non blocking)
loop.create_task(initialize_server(loop, queue))
# creating a new message (non blocking)
loop.create_task(telethon_telegram_generate(loop, queue, 'non blocking'))
# creating a new message (blocking)
loop.run_in_executor(None, telethon_telegram_generate, loop, queue, 'blocking')
# managing the queue
while True:
message = await queue.get()
print("Sending '{}'...".format(message))
await telegram_client.send_message(RECIPIENT, message)
time.sleep(1)
asyncio.run(main())
In this case, to trigger the generation of a new Telegram message from the TCP server itself, it is needed to launch a curl command like the following one:
curl http://localhost:8080
Nevertheless, results are not promising.
Results:
initialization: ok for telethon_telegram_init and initialize_server, apparently not working for both telethon_telegram_generate method calls
message reception: not working (no errors are returned)
message sending: not working (no errors are returned)
additional notes: after the curl command is executed, a New connection from 127.0.0.1:<random_port> message is printed, but no Telegram messages are sent

python websockets, multiple requests, why only requests are received?

I am trying to test out if I send out multiple requests at the same moment using coroutine can cause the server side receives corrupted data.
My test is based on the sample code from: https://websockets.readthedocs.io/en/stable/intro.html
Somehow, for the following code, the server side only receive one requests? Anyone has some insights? thx
server (this is basically the same code from the websockets Getting Started webpage):
#!/usr/bin/env python3
# WS server example
import asyncio
import websockets
async def hello(websocket, path):
name = await websocket.recv()
print(f"< {name}")
greeting = f"Hello {name}!"
await websocket.send(greeting)
print(f"> {greeting}")
start_server = websockets.serve(hello, "localhost", 8765)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
Client, I created 1000 tasks, and schedule to run them as soon as possible:
#!/usr/bin/env python3
# WS client example
import asyncio
import websockets
uri = "ws://localhost:8765"
connection = None
async def hello():
global connection
name = "What's your name? "
await connection.send(name)
print(f"> {name}")
async def main():
global connection
connection = await websockets.connect(uri)
#asyncio.run(main())
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.run_until_complete(asyncio.wait(
[hello() for i in range(1000)], return_when=asyncio.ALL_COMPLETED
))
UPDATE
The solution is to use a loop.

I found the reason: the server side, handler should use a loop so that the corroutine will not finish immediately after received the first request.

The documentation you linked also includes this paragraph just below the server code:
On the server side, websockets executes the handler coroutine hello once for each WebSocket connection. It closes the connection when the handler coroutine returns.
The client code you linked creates one connection and sends messages on that connection. After the client sends the first message, the server closes the connection, so the next 999 messages you attempt to send are being sent on a closed connection.
If you update the hello handler to include a loop, you will see all messages.
import asyncio
import websockets
async def hello(websocket, path):
while True:
name = await websocket.recv()
print(f"< {name}")
greeting = f"Hello {name}!"
await websocket.send(greeting)
print(f"> {greeting}")
start_server = websockets.serve(hello, "localhost", 8765)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()

Python Asyncio & Websocket how to avoid global variable?

I have a python socket server using asyncio and websockets. When the websocket is active 100+ devices will connect and hold their connection waiting for commands/messages.
There are two threads the first thread accepts connections and adds their details to a global variable then waits for messages from the device:
async def thread1(websocket, path):
client_address = await websocket.recv()
CONNECTIONS[client_address] = websocket
async for message in websocket:
... do something with message
start_server = websockets.serve(thread1, host, port)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.ensure_future(thread2())
asyncio.get_event_loop().run_forever()
The second thread processes some user data and once it needs to send a command it accesses a global variable to get the websocket info:
thread2()
...some data processing
soc = CONNECTIONS[ipaddress]
await soc.send("some message")
My question: What's the best way to allow another thread to send messages?
I can keep the global variable safe using thread locking and a function made only to process that data, however global variables aren't ideal. I cannot send information between threads since thread1 is stuck waiting to receive messages.

The first thing I would like to say is the incorrect use of the term thread. You use asyncio and here the concept is used - coroutine (coroutine is wrapped into a asyncio task). How it differs from threads can be found, for example, here.
The websockets server spawns a new task for each incoming connection (there are the same number of connections and spawned tasks). I don't see anything wrong with the global object, at least in a small script. However, below I gave an example where I placed this in a separate class.
Also, in this case, special synchronization between coroutines is not required, since they are implemented through cooperative multitasking (in fact, all are executed in one thread, transferring control at certain points.)
Here is a simple example in which the server stores a dictionary of incoming connections and starts a task that every 2 seconds, notifies all clients and sends them the current time. The server also prints confirmation from clients to the console.
# ws_server.py
import asyncio
import websockets
import datetime
class Server:
def __init__(self, host, port):
self.host = host
self.port = port
self.connections = {}
self.is_active = False
self.server = None
async def start(self):
self.is_active = True
self.server = await websockets.serve(self.handler, self.host, self.port)
asyncio.create_task(self.periodic_notifier())
async def stop(self):
self.is_active = False
self.server.close()
await self.wait_closed()
async def wait_closed(self):
await self.server.wait_closed()
async def handler(self, websocket, path):
self.connections[websocket.remote_address] = websocket
try:
async for message in websocket:
print(message)
except ConnectionClosedError as e:
pass
del self.connections[websocket.remote_address]
print(f"Connection {websocket.remote_address} is closed")
async def periodic_notifier(self):
while self.is_active:
await asyncio.gather(
*[ws.send(f"Hello time {datetime.datetime.now()}") for ws in self.connections.values()],
return_exceptions=True)
await asyncio.sleep(2)
async def main():
server = Server("localhost", 8080)
await server.start()
await server.wait_closed()
asyncio.run(main())
# ws_client.py
import asyncio
import websockets
async def client():
uri = "ws://localhost:8080"
async with websockets.connect(uri) as websocket:
async for message in websocket:
print(message)
await websocket.send(f"ACK {message}")
asyncio.run(client())

Running coroutines concurrently not working properly in pythong/asyncio

I have a publisher/subscriber architecture running on my websocket server, where the publisher runs in one thread, and the websocket server in another. I connect to the server from the publisher over localhost, and the server distributes the published messages to any other connected clients on the /sub path. However, since the publisher thread not always has new data to publish, it has a tendency to disconnect after a timeout of 50 sec. To fix this, I implemented a heartbeat ping function:
async def ping(websocket):
while True:
await asyncio.sleep(30)
print("[%s] Pinging server..." % datetime.now())
await websocket.send('ping')
This keeps the publisher from disconnecting. However, when I'm trying to run this concurrently with the coroutine that sends the actual data, I cannot get both ping() and send_data() to run in parallel. I've tried just awaiting both functions as well as asyncio.gather() (which according to documentation is supposed to run tasks concurrently) as well as flipping the order, but it seems like in all cases only the first function call is ran.
My thread class for reference:
class Publisher(threading.Thread):
"""
Thread acting as the websocket publisher
Pulls data from the data merger queue and publishes onto the websocket server
"""
def __init__(self, loop, q, addr, port):
threading.Thread.__init__(self)
self.loop = loop
self.queue = q
self.id = threading.get_ident()
self.addr = addr
self.port = port
self.name = 'publisher'
print("Publisher thread started (ID:%s)" % self.id)
def run(self):
self.loop.run_until_complete(self.publish())
asyncio.get_event_loop().run_forever()
async def ping(self, websocket):
while True:
await asyncio.sleep(30)
print("[%s] Pinging server..." % datetime.now())
await websocket.send('ping')
async def send_data(self, websocket):
while True:
try:
msg = json.dumps(self.queue.get()) # Get the data from the queue
print(msg)
await asyncio.sleep(0.1)
if not msg:
print("No message")
break
await websocket.send(msg)
except websockets.exceptions.ConnectionClosedError:
print("Connection closed")
break
async def publish(self):
uri = 'ws://' + str(self.addr) + ':' + str(self.port) + '/pub'
async with websockets.connect(uri) as websocket:
await asyncio.gather(
self.ping(websocket),
self.send_data(websocket)
)

asyncio.subprocess always blocks

I need to write some async code which runs a subprocess as part of its tasks. Even though I am using asyncio.subprocess my code is still blocking. My server looks like this:
import asyncio
import asyncio.subprocess
import websockets
async def handler(websocket, path):
while True:
data = await websocket.recv()
print('I received a message')
player = await asyncio.create_subprocess_exec(
'sleep', '5',
stdin=asyncio.subprocess.DEVNULL,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL)
await player.wait()
print('Finished waiting')
server = websockets.serve(handler, '0.0.0.0', '8000')
asyncio.get_event_loop().run_until_complete(server)
asyncio.get_event_loop().run_forever()
And a very basic client:
import asyncio
import websockets
async def client():
async with websockets.connect('ws://localhost:8000') as websocket:
for i in range(5):
await websocket.send('message')
await asyncio.sleep(0.5)
asyncio.get_event_loop().run_until_complete(client())
I would expect the output to look like this:
I received a message
I received a message
I received a message
I received a message
I received a message
Finished waiting
Finished waiting
Finished waiting
Finished waiting
Finished waiting
But instead I get this:
I received a message
Finished waiting
I received a message
Finished waiting
I received a message
Finished waiting
I received a message
Finished waiting
I received a message
Finished waiting
With a 5 second wait after each "I received a message" line.

The line await player.wait() does not block other async operations, but waits for 5 seconds!
If you don't want to wait for the response, try using ensure_future() instead:
# add:
async def wait_for_player(player, path):
print("Waiting...", path)
await player.wait()
print("Done", path)
# and replace await player.wait() with:
asyncio.ensure_future(wait_for_player(player, path))
You can actually also move create_subprocess_exec() to wait_for_player().
To see your code is not blocking see try these:
Client:
import asyncio
import websockets
async def client(n):
async with websockets.connect('ws://localhost:8000/{}/'.format(n)) as websocket:
print(n, "start")
for i in range(5):
print(n, i)
await websocket.send('message')
await asyncio.sleep(0.5)
print(n, "done")
tasks = [client(i) for i in range(5)]
asyncio.get_event_loop().run_until_complete(asyncio.wait(tasks))
Server:
import asyncio
import asyncio.subprocess
import random
import websockets
async def handler(websocket, path):
try:
while True:
data = await websocket.recv()
pause = random.randint(1, 5)
print('I received a message', path, "Pausing:", pause)
player = await asyncio.create_subprocess_exec(
'sleep', str(pause),
stdin=asyncio.subprocess.DEVNULL,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL)
await player.wait()
print('Finished waiting', path)
except websockets.ConnectionClosed:
print("Connection closed!", path)
server = websockets.serve(handler, '0.0.0.0', '8000')
asyncio.get_event_loop().run_until_complete(server)
asyncio.get_event_loop().run_forever()

Your ws server seems ok. Actually it is your client that is blocking. If you want to test the async behavior of your server, You need to make asynchronous requests. The for loop in your client blocks the thread. So remove it and instead, use asyncio.gather to run your client() method 5 times asynchronously
import asyncio
import websockets
async def client():
async with websockets.connect('ws://localhost:8000') as websocket:
await websocket.send('message')
await asyncio.sleep(0.5)
tasks = asyncio.gather(*[client() for i in range(5)])
asyncio.get_event_loop().run_until_complete(tasks)