I have this code:
import sys
import socket
import asyncio
async def main(dest_addr, max_hops=30, timeout=0.5):
loop = asyncio.get_event_loop()
queue = asyncio.Queue()
port = 33434
rx = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP)
rx.settimeout(timeout)
rx.bind(("", port))
def reader():
try:
_, addr = rx.recvfrom(512)
addr = addr[0]
except socket.timeout:
addr = None
queue.put_nowait(addr)
loop.add_reader(rx, reader)
tx = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
for ttl in range(1, max_hops + 1):
tx.setsockopt(socket.SOL_IP, socket.IP_TTL, ttl)
tx.sendto(b"", (dest_addr, port))
addr = await queue.get()
print(ttl, addr)
loop.remove_reader(rx)
if __name__ == "__main__":
dest_name = sys.argv[1]
dest_addr = socket.gethostbyname(dest_name)
print(f"traceroute to {dest_name} ({dest_addr})")
asyncio.get_event_loop().run_until_complete(main(dest_addr))
I am basically trying to implement traceroute using asyncio.
I'm monitoring the socket file descriptor for read availability and invoke reader when I receive data from a device after using the socket.sendto method and wait for the queue to be filled before going to the next step.
However, I my program hangs right after the first iteration, on the second addr = await queue.get().
It seems like the reader callback is only invoked once and never again so the queue is not filled, which is strange because I have a timeout of 0.5s on the rx socket.
Answering my own question:
I think what happens is that, the device (my front router for example) doesn't respond anything so I am never notified when the file descriptor is ready for reading, so the callback is not invoked.
The workaround is to wrap the queue.get() within asyncio.wait_for with a timeout so it doesn't hang forever:
for ttl in range(1, max_hops + 1):
tx.setsockopt(socket.SOL_IP, socket.IP_TTL, ttl)
tx.sendto(b"", (dest_addr, port))
try:
addr = await asyncio.wait_for(queue.get(), timeout)
except asyncio.TimeoutError:
addr = "timeout"
print(ttl, addr)
Related
I was writing an echo server but recv() does not return while the sender process is alive. If I use one recv() call instead of my recvall() function, it is returning and working. Please let me know what is wrong with recvall() function..
Server:
import socket
def recvall(skt):
msg_total = b""
while True:
msg = skt.recv(1024)
if not msg: break
msg_total += msg
return msg_total
def main():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("localhost",3333))
s.listen()
conn,addr = s.accept()
# data = conn.recv(1024) <= this recv() is not blocking
data = recvall(conn)
conn.sendall(data)
main()
Client:
import socket
def main():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("localhost",3333))
msg = ("a" * 10).encode()
s.sendall(msg)
rsp = s.recv(1024)
main()
Is there something the client needs to do in order to flag that "This is the end?"
For example, I have the following server running:
import socket, sys
while True:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('', 5500))
s.listen(1)
conn, addr = s.accept()
print('Connected by', addr)
data = b''
while True:
data_chunk = conn.recv(10)
data += data_chunk
if not data: break
# How do I get here so I can echo the message *at the end* ?
# conn.send(b'Done. What you sent was: %s' % data)
And I have the following client code:
import socket
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client_socket.connect(('sandbox2.myip.com', 5500))
client_socket.send(b'select * from table')
# Now, what to do so that I can get the final message from he server?
When I do the connection though, this is what it shows:
sampleDB$ python listener.py
Connected by ('99.51.149.29', 63352)
I can confirm that the data data_chunks are being received, but then it just seems to hang at this part right here:
data_chunk = conn.recv(10)
Is there something I need to do in the client (or the server?) to flag that "This is the end of the transmission, so conn.recv() doesn't just hang indefinitely?
Here's a complete example of a socket based client/server protocol that implements the strategy described in my comment:
import socket
import threading
from struct import pack, unpack
HOST = 'localhost'
PORT = 7070
FORMAT = ('!Q', 8)
MSG = '''The Owl and the Pussy-cat went to sea
In a beautiful pea-green boat,
They took some honey, and plenty of money,
Wrapped up in a five-pound note.
The Owl looked up to the stars above,
And sang to a small guitar,
"O lovely Pussy! O Pussy, my love,
What a beautiful Pussy you are,
You are,
You are!
What a beautiful Pussy you are!"'''
def sendbuffer(s, b):
buffer = pack(FORMAT[0], len(b)) + b
offset = 0
while offset < len(buffer):
offset += s.send(buffer[offset:])
def recvbuffer(s):
p = s.recv(FORMAT[1], socket.MSG_WAITALL)
n = unpack(FORMAT[0], p)[0]
return None if n == 0 else s.recv(n, socket.MSG_WAITALL)
def server():
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind((HOST, PORT))
s.listen()
conn, _ = s.accept()
with conn:
while (data := recvbuffer(conn)):
sendbuffer(conn, data)
def client(msg):
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((HOST, PORT))
sendbuffer(s, msg.encode())
data = recvbuffer(s)
print(data.decode())
sendbuffer(s, b'') # kill switch
def main():
threading.Thread(target=server).start()
client(MSG)
if __name__ == '__main__':
main()
In an attempt to solve this, I'm trying to simplify the problem. Suppose I have a receiver listening to both TCP and UDP messages. It will receive several strings, append them to a deque and after receiving "finish" message, it will start processing the deque.
If I receive a UDP message, I need to stop the processing, remove the last item of deque and then continue the processing.
from collections import deque
host = commands.getoutput("hostname -I")
port = 5005
backlog = 5
BUFSIZE = 4096
q = deque()
def read_tcp(s):
conn, addr = s.accept()
print('Connected with', *addr)
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print "received data:", data
conn.send(data) # echo
conn.close()
if (data == 'finish'):
processP(q)
else:
q.append(data)
def read_udp(s):
data,addr = s.recvfrom(1024)
print("received message:", data)
del q[-1]
processP(q):
text = q.popleft()
textReverse = text[::-1]
print(textReverse)
def run():
# create tcp socket
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
tcp.bind((host,port))
except socket.error as err:
print('Bind failed', err)
return
tcp.listen(1)
# create udp socket
udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
udp.bind((host,port))
print('***Socket now listening at***:', host, port)
input = [tcp,udp]
try:
while True:
inputready,outputready,exceptready = select(input,[],[])
for s in inputready:
if s == tcp:
read_tcp(s)
elif s == udp:
read_udp(s)
else:
print("unknown socket:", s)
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
raise
tcp.close()
udp.close()
if __name__ == '__main__':
run()
I have problem in pausing the program upon receiving a UDP message and then returning to the processing phase. Right now, if a UDP message is sent to my program while processing, it won't receive the message until the end of processing (and then the deque is empty). I thought maybe threading or multiprocessing may help, but I can't figure out how to apply them to the code.
Nobody forces you to empty out the dequeue. You can check if an UDP message has arrived before dequeueing the next workload. And that is as far as you can get with threads, as they do not allow you to interrupt arbitrary code. They can always only be terminated cooperatively.
If your single item processing takes too long, then multiprocessing of work-items is an option, as you can kill an external process.
Use select.select to check for incoming data on your sockets with a short timeout, before continuing to process the next workload. Alternatively you could use a thread waiting on input on the thread and manipulate the dequeue.
EDIT This is your code made to work with python3, select.select and a timeout. Triggering read_udp works with netcat with echo foo | nc -4 -u localhost 5005 but then triggers an exception because you assume the existence of elements in the dequeue - which is an application logic problem that is independent of the question how to interleave listening and working.
import socket
import select
from collections import deque
host = "localhost"
port = 5005
backlog = 5
BUFSIZE = 4096
q = deque()
def read_tcp(s):
conn, addr = s.accept()
print('Connected with', *addr)
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.send(data) # echo
conn.close()
if (data == 'finish'):
processP(q)
else:
q.append(data)
def read_udp(s):
data,addr = s.recvfrom(1024)
print("received message:", data)
del q[-1]
def processP(q):
text = q.popleft()
textReverse = text[::-1]
print(textReverse)
def run():
# create tcp socket
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
tcp.bind((host,port))
except socket.error as err:
print('Bind failed', err)
return
tcp.listen(1)
# create udp socket
udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
udp.bind((host,port))
print('***Socket now listening at***:', host, port)
input = [tcp,udp]
try:
while True:
print("select.select")
inputready,outputready,exceptready = select.select(input,[],[], 0.1)
for s in inputready:
if s == tcp:
read_tcp(s)
elif s == udp:
read_udp(s)
else:
print("unknown socket:", s)
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
raise
tcp.close()
udp.close()
if __name__ == '__main__':
run()
So I have made some code that follows. It is suppose to let a server and a client communicate... but it doesn't work.
Can someone explain why, or better yet fix my code???
Server.
import time
import socket
from threading import Thread
global sS
sS = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sS.bind(('', 2347))
sSconAddresses = []
sSconData = []
print(" Server : Running ... ")
sS.listen(10)
while True:
try:
cOn, aDr = sS.accept()
sSconAddresses.insert(0, str(aDr))
sSconData.insert(0, str(cOn))
time.sleep(0.3)
except:
time.sleep(0.1)
pass
def ConHandler():
for _ in sSconData:
PacketData = _.recv(700)
if not PacketData:
_.close()
else:
stringData = PacketData.decode('utf-8')
print(stringData)
sS.sendto(PacketData, _)
ConHandlerThread = Thread(target=ConHandler)
ConHandlerThread.daemon = True
ConHandlerThread.start()
Client.
import threading, time
import socket, sys
import os
global cS
cS = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
cS.connect(('PRIVATE', 2347))
Server = ('PRIVATE', 2347)
while True:
PacketData = input(" Client> ")
ByteData = PacketData.encode('utf-8')
cS.sendto(ByteData, Server)
It doesn't return any errors so I am confused why it doesn't work.
First of all, in your server-side code, you're having a while True before starting your thread, so it can't work.
Then, if you succeed starting your thread by moving the code, its for will see an empty list, so it will not loop, and just exit right here.
Starting from your code, here's something that works:
The client:
import socket
def main():
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client_socket.connect(('127.0.0.1', 2345))
while True:
packetdata = input(" Client> ")
bytedata = packetdata.encode()
client_socket.send(bytedata)
print(client_socket.recv(700).decode())
if __name__ == '__main__':
main()
The server:
import socket
from threading import Thread
from queue import Queue
def client_handler(client_socket):
while True:
data = client_socket.recv(700)
print(data)
client_socket.send("Server {}".format(data.decode()).encode())
def conn_handler(conn_queue):
while True:
conn, address = conn_queue.get()
print("Handler getting a connection from {}".format(address))
client_thread = Thread(target=client_handler, args=(conn,))
client_thread.daemon = True
client_thread.start()
def main():
print("Server: Running ...")
conn_queue = Queue()
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind(('', 2345))
server_socket.listen(10)
con_handler_thread = Thread(target=conn_handler, args=(conn_queue,))
con_handler_thread.daemon = True
con_handler_thread.start()
while True:
conn_queue.put(server_socket.accept())
if __name__ == '__main__':
main()
Note that this is suboptimal, starting one thread per client is not the way to go. The best way to handle this kind of situation is to keep everything in a single thread and use a select-like function to know what to do. However select is a bit limited too (like 1024 connections max, hardcoded in the libc), so the way to go is to use epoll / kqueue / whatever better than poll / select, and there's a module for this: https://docs.python.org/3/library/select.html
Yet using the select module is still the old, manual, cubersome way to express your needs, you should take a look at the coroutine based API of asyncio which enable a clear way to express the intention.
The asyncio equivalent may look like:
import asyncio
async def client():
reader, writer = await asyncio.open_connection('127.0.0.1', 8888)
while True:
message = input("Client> ")
writer.write(message.encode())
data = await reader.read(100)
print('Received: {}'.format(data.decode()))
loop = asyncio.get_event_loop()
loop.run_until_complete(client())
And, server side:
import asyncio
async def handle_client(reader, writer):
while True:
data = await reader.read(100)
if not data:
return
message = data.decode()
addr = writer.get_extra_info('peername')
print("Received %r from %r" % (message, addr))
print("Send: %r" % message)
writer.write(data)
await writer.drain()
loop = asyncio.get_event_loop()
coro = asyncio.start_server(handle_client, '127.0.0.1', 8888)
server = loop.run_until_complete(coro)
print('Serving on {}'.format(server.sockets[0].getsockname()))
loop.run_forever()
I have a program interacting with two servers (all on localhost), which essentially forwards data from one server to the other.
I am trying to create a scenario where the server sending the initial data will close its connection to the middle-program because the server receiving the data has a very slow connection to the middle-program. The buffers of the middle-program should get filled, and it shouldn't be able to receive any data from the server sending data. As a result of this inactivity, the server sending data should timeout and close the connection, resulting in an in-complete data transfer.
I am trying to do it in the following way:
Program at server sending the data
import socket
interval_sec = 3
TCP_KEEPALIVE = 0x10
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
sock.setsockopt(socket.IPPROTO_TCP, TCP_KEEPALIVE, interval_sec)
sock.bind(('', 9000))
sock.listen(5)
while True:
conn, addr = sock.accept()
f = open('test.txt', 'r')
file_data = f.read(1024)
while (file_data):
conn.send(file_data)
file_data = f.read(1024)
f.close()
print "sent"
Middle program forwarding the data
import socket
import select
import time
import struct
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
recv_timeout = 2
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO, struct.pack('LL', recv_timeout, 0))
total_data=[]
total_sent=0;
data=0;
sock.connect(('localhost', 9000))
sock2.connect(('localhost', 8001))
sock2.setblocking(0)
sock.setblocking(0)
j=0
while 1:
read, write, errors = select.select([sock], [sock2], [], 2)
if read:
data=sock.recv(1024, socket.MSG_DONTWAIT)
total_data.append(data)
#time.sleep(10)
if write:
if data:
try:
data_sent=sock2.send(data, socket.MSG_DONTWAIT)
total_sent+=data_sent
data=data[data_sent:]
print data_sent
except socket.error, e:
if e.errno != errno.EAGAIN:
raise e
else:
print "Not writable"
Program at server receiving the data
import socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', 8001))
sock.listen(5)
while True:
conn, addr = sock.accept()
with open('received_file', 'wb') as f:
while True:
file_data = conn.recv(100)
print('receiving data...')
if not file_data:
break
f.write(file_data)
f.close()
conn.close()
The problem I'm facing is that, the middle-program's sockets are blocking although they should be acting as non-blocking.
Are my programs fundamentally wrong, and I'm missing the whole point or do I need to make some minor tweaks?
Thanks :)