Develop Reference

Python UDP socket receive - server.recvfrom package size longer than bufsize

I'm using the socket module for a UDP server。The incoming packets always have a different size(0-65535), so client send package length first, then send the package；server receives data according to the package length and 1024 per server receives。but server didn't work like I thought，it can receives package only once。
# -*- coding: utf-8 -*-
import json
import socket
ip_port = ('127.0.0.1', 8080)
server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server.bind(ip_port)
print('server listen......')
def recv_header(s_socket):
try:
msg, addr = s_socket.recvfrom(32)
return json.loads(msg), addr
except Exception as e:
return str(e), None
while True:
header_msg, client_addr = recv_header(s_socket=server)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
receiverBufsize = 1024
if client_addr is None:
print (header_msg)
continue
data_body = ""
package_len = int(header_msg['length'])
print ("package_len: {}".format(package_len))
print ("client_addr: {}".format(str(client_addr)))
while package_len > 0:
if package_len > receiverBufsize:
body_part = server.recvfrom(receiverBufsize)
else:
body_part = server.recvfrom(package_len)
data_body += body_part[0]
package_len -= receiverBufsize
print ("data_body: {}".format(data_body))
# -*- coding: utf-8 -*-
import json
import socket
import random
ip_port = ('127.0.0.1', 8080)
client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def get_data(length):
base_str = 'ABCDEFGHIGKLMNOPQRSTUVWXYZabcdefghigklmnopqrstuvwxyz0123456789'
return ''.join([random.choice(base_str) for _ in range(length)])
while 1:
msg = raw_input('>>>：')
if msg == 'q':
break
if not msg:
continue
len = int(msg)
client.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
data = get_data(len)
header = json.dumps({"length": len})
aa = client.sendto(header.encode('utf-8'), ip_port)
print aa
aa = client.sendto(data.encode('utf-8'), ip_port)
print aa
If the packet length is less than 1024(my receiverBufsize), it works normally
enter image description here
enter image description here
If the packet length is more than 1024(my receiverBufsize), socket.recvfrom will block
enter image description here
enter image description here

UDP is a message oriented protocol, not a data stream like TCP. This means in UDP a single send is matched by a single recv.
This means you cannot use multiple recv with a small buffer (size 1024 in your case) to get data which were sent in a single send with a larger buffer - you need to use recv with a large enough buffer instead.
This means also that you don't actually need to send the length upfront, since you don't need to add message semantics to a protocol which already has message semantic. In contrary, sending the length in a separate message as you do could actually be harmful since UDP is an unreliable protocol and you cannot rely on packets getting delivered in order, at all or only once.

Python network scanner (host discovery tool) is telling me that my source and destination are the same when sending UDP datagrams

I'm following Black Hat Python (2ed.), in which I'm writing a network scanning tool. The tool is in theory supposed to send UDP packets out to a given subnet, and if a host is up on that subnet, the response packet is decoded, found to contain the message in the original datagram, and used to indicate the host is up. This seems to generally be working well to capture packets; I can go to a website, or ping another host, and the tool reliably provides the correct source and destination addresses for those cases.
Here is the meat of the code (I have not included the class creation, or the passing of the host argument for brevity, but the host is 192.168.10.85).
class IP:
"""layer 3 (IP) packet header decoder"""
def __init__(self, buff=None):
header = struct.unpack('<BBHHHBBH4s4s', buff)
self.ver = header[0] >> 4
self.ihl = header[0] & 0xF
self.tos = header[1]
self.len = header[2]
self.id = header[3]
self.offset = header[4]
self.ttl = header[5]
self.protocol_num = header[6]
self.sum = header[7]
self.src = header[8]
self.dst = header[9]
# make IP addrs human readable
self.src_address = ipaddress.ip_address(self.src)
self.dst_address = ipaddress.ip_address(self.dst)
# the protocol_num is actually a code for the protocol name
self.protocol_name = {1: 'ICMP', 6: 'TCP', 17: 'UDP'}
# try to provide the human version of the protocol, otherwise just give the code
try:
self.protocol = self.protocol_name[self.protocol_num]
except KeyError as error:
self.protocol = self.protocol_num
print(f'Protocol is unrecognized, try googling "IP protocol {self.protocol_num}"')
class ICMP:
"""layer 4 (ICMP) packet header decoder"""
def __init__(self, buff):
header = struct.unpack('<BBHHH', buff)
self.type = header[0]
self.code = header[1]
self.checksum = header[2]
self.ident = header[3]
self.seq_num = header[4]
def udp_sender():
# blasts udp packets into the network to solicit responses
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sender:
for ip in ipaddress.ip_network(SUBNET).hosts():
# time.sleep(1)
print(f'sending a test message to {ip}')
# send our test message out to port 65212 on the destination
sender.sendto(bytes(MESSAGE, 'utf8'), (str(ip), 65212))
class Scanner:
def __init__(self, host):
self.host = host
# create raw socket, bind to public interface
# if windows:
if os.name == 'nt':
socket_protocol = socket.IPPROTO_IP
# if linux/mac:
else:
socket_protocol = socket.IPPROTO_ICMP
self.socket = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket_protocol)
self.socket.bind((host, 0))
# socket options, include header
self.socket.setsockopt(socket_protocol, socket.IP_HDRINCL, 1)
# enable promiscuous mode for windows
if os.name == 'nt':
self.socket.ioctl(socket.SIO_RCVALL, socket.RCVALL_ON)
def sniff(self):
# set of all hosts that are up (respond to our ICMP message)
hosts_up = {f'{str(self.host)} *'}
try:
while True:
# read a packet, and parse the IP header
raw_buffer = self.socket.recvfrom(65535)[0]
# create IP header from the first 20 bytes
ip_header = IP(raw_buffer[0:20])
# if the protocol is ICMP, do some additional things
# print(f'src={ip_header.src_address}, dst={ip_header.dst_address}, prot_name={ip_header.protocol}')
if ip_header.protocol == 'ICMP':
# calculate where the ICMP packet starts
offset = ip_header.ihl * 4
buf = raw_buffer[offset:offset + 8]
# create ICMP structure
icmp_header = ICMP(buf)
print(f'type: {icmp_header.type}, code: {icmp_header.code}')
print(f'src={ip_header.src_address}, dst={ip_header.dst_address}, prot_name={ip_header.protocol}')
if icmp_header.type == 3 and icmp_header.code == 3:
print(f'type: {icmp_header.type}, code: {icmp_header.code}')
print(f'src={ip_header.src_address}, dst={ip_header.dst_address}, prot_name={ip_header.protocol}')
if ipaddress.ip_address(ip_header.src_address) in ipaddress.IPv4Network(SUBNET):
# make sure the packet has our test message
if raw_buffer[len(raw_buffer) - len(MESSAGE):] == bytes(MESSAGE, 'utf8'):
tgt = str(ip_header.src_address)
if tgt != self.host and tgt not in hosts_up:
hosts_up.add(str(ip_header.src_address))
print(f'Host Up: {tgt}')
However, when receiving the ICMP responses as a result of my datagram, the tool reports that the source and destination addresses are the same (my host, 192.168.10.85). Furthermore, while I should be receiving responses with Type 3 and Code 3 (destination unreachable, and port unreachable), but I am receiving (in my program) Type 3 and Code 1.
Here is an example of the output when I issue a ping command while the scanner is running, which seems correct:
src=192.168.10.85, dst=192.168.10.200, prot_name=ICMP type: 0, code: 0 src=192.168.10.200, dst=192.168.10.85, prot_name=ICMP type: 8, code: 0
Here is an example of the output to what I am assuming is the UDP packet response, which seems incorrect):
src=192.168.10.85, dst=192.168.10.85, prot_name=ICMP type: 3, code: 1
If I open wireshark while I'm running my code, I can correctly see the ICMP Type 3/Code 3 responses, so I know they are going through, here is a screen grab of one host on the target subnet as an example:
Why is my scanner not seeing these responses that are in wireshark?
I've tried running wireshark alongside my program, to see if the packets are being correctly decoded, and that the message in the UDP packet is properly in place. All signs indicate that the packets are going out to the hosts I'm trying to detect, and the correct responses are coming back, but my scanner refuses to find them.

How to concatenate 3 separate UDP packets?

This question has probably been asked before but I can't seem to find it.
I am trying to send 3 individual packets between a client and a server. That part I can do, however I need to take those 3 packets and add them together and print the result.
That's the part I am having trouble with. I have never programmed before and have only really started so I am new to all this and please be gentle with what is probably awful coding on my part!
This is what I have for the client:
import socket
clientSock = socket.socket(socket.AF_INET , socket.SOCK_DGRAM)
UDP_IP = "localhost"
UDP_PORT = 6842
address = ("localhost" , 6842)
s1 = str(input("Please enter Packet number 1: "))
clientSock.sendto(bytes("s1", "utf-8"), (address))
s2 = str(input("Please enter Packet number 2: "))
clientSock.sendto(bytes("s2", "utf-8"), (address))
s3 = str(input("please enter Packet number 3: "))
clientSock.sendto(bytes("s3", "utf-8"), (address))
print ("Sent 3 packets to server")
print ("Waiting to receive message...")
And this is what I have for the server:
import socket
serverSocket = socket.socket(socket.AF_INET , socket.SOCK_DGRAM)
UDP_IP = "localhost"
UDP_PORT = 6842
address = ("localhost" , 6842)
serverSocket.bind(address)
print ("Waiting for client...")
while True:
data,addr = serverSocket.recvfrom(6842)
print ("Received:",data," from",addr)
n = len('s1' , 's2' , 's3')
R = "s1" + "#" + "s2" + "#" + "s3"
print (n)
print (R)
Again there are probably quite a few flaws in this but was wondering if anyone can point me in the right direction.

I think your server just wants to do something closer to:
while True:
values = []
for i in range(3):
values.append(float(sock.recv(1024)))
print(sum(values))
note that UDP is unreliable, so you might not always receive the three packets sent from the client
you can also compress the contents of the while loop into a single line, and hence avoid creating any list, but I'll leave that up to you!

Python Server-Client Lottery Issue

I am have an issue regarding a python assignment I was doing and would like to ask the community for some guidance. I am to use the socket module and argparse modules only for this assignment. I have created a socketserver.py file and a socketclient.py file. The connection between the server and client is fine. Now the purpose of the assignment is that the client is to send the Lottery game type, # of tickets, and # of lines per tickets using argparse. E.g. syntax for socketclient.py would be python3 -t Lotto_Max -n 2 -l 2. The output for the ticket game, ticket type and number of lines per ticket show up correctly on the server. However, they sometimes don't show correctly on the client, and am really stuck at the moment. Here is my following code....
Server Code
```socketserver.py code```
import socket
from random import sample
def main():
host = input("Please specify an IP address for this server's socket\t")
port = int(input('Please speciy a port number above 1024 for this socket.\t'))
kulmiye = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
kulmiye.bind((host, port))
except socket.error as egeh:
print('Socket bind failed. Error Code : ' + str(egeh[0]) + ' Message ' + egeh[1])
print('Socket bind accomplished...\n')
print("Listening for an incoming connection...")
kulmiye.listen()
conn, addr = kulmiye.accept()
print('Connected with ' + addr[0] + ':' + str(addr[1]))
while True:
server_data = conn.recv(1024).decode("utf-8")
game_data = server_data.split(",")
if not server_data:
break
if game_data[0] == "Lotto_Max":
nval = int(game_data[1])
lval = int(game_data[2])
for nval in range(nval):
for i in range(lval):
numbers = sample(range(1, 50), 7)
numbers.sort()
sortedd = str(numbers)
print(sortedd)
print("--------------------")
conn.sendall(sortedd.encode("utf-8"))
#conn.sendall(bytes(str(numbers),'utf-8'))
liners = "-----------------------"
conn.sendall(liners.encode("utf-8"))
print("From Client: " + str(game_data))
conn.sendall(b'goodbye')
# server_data = input('#\t')
break
else:
conn.close()
if __name__ == '__main__':
main()
Client Code
```socketclient.py code```
import socket
import argparse
def client():
host = input("Please specify the server's IP you want to connect to\t")
port = int(input("Please specify the server's port you want to connect to\t"))
kulmiye = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
kulmiye.connect((host, port))
# client_data = input("#\t")
# while True:
# client_data = input()
# if client_data == 'quit':
# kulmiye.close()
# sys.exit()
# if len(str.encode(client_data)) > 0:
# kulmiye.sendall(str.encode(client_data))
# server_response = str(kulmiye.recv(1024), "utf-8")
# print(server_response, end = " ")
kulmiye.sendall(bytes(tvar.encode("utf-8")))
kulmiye.sendall(bytes(','.encode("utf-8")))
kulmiye.sendall(bytes(str(nval).encode("utf-8")))
kulmiye.sendall(bytes(','.encode("utf-8")))
kulmiye.sendall(bytes(str(lval).encode("utf-8")))
server_data = kulmiye.recv(1024).decode("utf-8")
while server_data != 'goodbye':
server_data = kulmiye.recv(1024).decode("utf-8")
print('Server: \n' + server_data)
# client_data = input("#\t")
if not server_data:
break
kulmiye.close()
# this code block serves to give the user the ability to play lotto max
# with the amount of tickets and lines per ticket they would like
# Using the argparse module to allow the user to input command-line interfaces
parser = argparse.ArgumentParser(description='Welcome to OLG Gaming.')
parser.add_argument(
'-t',
type=str,
help="Pick the lottery you want to play",
required=True)
parser.add_argument(
'-n',
type=int,
help="Pick the amount of lottery tickets you want to play",
required=True)
parser.add_argument(
'-l',
type=int,
help="Pick the amount of lines you would like to play",
required=True)
# parser.add_argument('-o', type = str, help = "This is optional", required = False)
# parse_args will convert the argument strings into objects and will get
# stored in the cmdargs variable
cmdargs = parser.parse_args()
tvar = cmdargs.t # the t string argument that gets parsed into an object will get stored into a variable called tvar
# the n integer argument that gets parsed into an object will get stored
# into a variable called nval
nval = int(cmdargs.n)
# the l integer argument that gets parsed into an object will get stored
# into a variable called lval
lval = int(cmdargs.l)
if __name__ == "__main__":
client()
```code```
Server
python3 socketserver.py
specify localhost as IP
specify a port e.g. 4444
Client
python3 socketclient.py -t Lotto_Max -n 1 -l 1
specify an IP address to connect to the server (e.g. localhost or 127.0.0.1)
specify a port to connect to e.g. 4444
When the connection establishes between client and server, the server receives the client input and prints it on its end the gametype (Lotto_Max), number of tickets and lines per ticket
Server will output the resultse.g.
However, the client won't receive it indefinitely. Usually it'll get it about 25% of the time, and I am not sure why

One problem is here in the server:
server_data = conn.recv(1024).decode("utf-8")
conn.recv(1024) can receive any number of bytes from 0 (closed connection) to 1024. The line above assumes the whole message is received every time, but when it fails it only gets part of the message. Below I've modified your code so the server can process multiple client connections one at a time, and the client will connect/disconnect over and over again to hasten the failure:
Server:
import socket
from random import sample
def main():
host = ''
port = 4444
kulmiye = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
kulmiye.bind((host, port))
print('Socket bind accomplished...\n')
print("Listening for an incoming connection...")
kulmiye.listen()
while True:
conn, addr = kulmiye.accept()
print('Connected with ' + addr[0] + ':' + str(addr[1]))
with conn:
server_data = conn.recv(1024).decode("utf-8")
print(f'server_data={server_data}')
game_data = server_data.split(",")
print("From Client: " + str(game_data))
if server_data:
if game_data[0] == "Lotto_Max":
nval = int(game_data[1])
lval = int(game_data[2])
for nval in range(nval):
for i in range(lval):
numbers = sample(range(1, 50), 7)
numbers.sort()
sortedd = str(numbers)
print(sortedd)
print("--------------------")
conn.sendall(sortedd.encode("utf-8"))
liners = "-----------------------"
conn.sendall(liners.encode("utf-8"))
conn.sendall(b'goodbye')
if __name__ == '__main__':
main()
Client:
import socket
import argparse
def client():
host = 'localhost'
port = 4444
kulmiye = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
kulmiye.connect((host, port))
kulmiye.sendall(bytes(tvar.encode("utf-8")))
kulmiye.sendall(bytes(','.encode("utf-8")))
kulmiye.sendall(bytes(str(nval).encode("utf-8")))
kulmiye.sendall(bytes(','.encode("utf-8")))
kulmiye.sendall(bytes(str(lval).encode("utf-8")))
server_data = kulmiye.recv(1024).decode("utf-8")
while server_data != 'goodbye':
server_data = kulmiye.recv(1024).decode("utf-8")
print('Server: \n' + server_data)
# client_data = input("#\t")
if not server_data:
break
kulmiye.close()
tvar = 'Lotto_Max'
nval = 1
lval = 1
if __name__ == "__main__":
while True:
client()
Here's the result after 100s of successful connections. Note server_data between the successful and unsuccessful connection:
Connected with 127.0.0.1:12175
server_data=Lotto_Max,1,1
From Client: ['Lotto_Max', '1', '1']
[4, 7, 9, 12, 24, 31, 48]
--------------------
Connected with 127.0.0.1:12176
server_data=Lotto_Max,1,
From Client: ['Lotto_Max', '1', '']
Traceback (most recent call last):
File "C:\server.py", line 38, in <module>
main()
File "C:\server.py", line 24, in main
lval = int(game_data[2])
ValueError: invalid literal for int() with base 10: ''
conn.recv(1024) didn't receive the complete message (didn't get the final 1). TCP is a byte streaming protocol with no concept of message boundaries. You're code is responsible to call recv() and buffer the data until you have a complete message. You could send fixed-sized messages and call recv until you have that many bytes, terminate a message with a sentinel byte such as a newline(\n) and read until the sentinel is received, or send the size of the message first followed by the message bytes.
I won't solve it for you since it is an assignment, but as a hint socket.makefile can wrap the socket in a file-like object where you can call .read(n) to read exactly n bytes from a socket, or .readline() to read a \n-terminated line.
Here's a link to an answer of mine that demonstrates this: https://stackoverflow.com/a/55840026/235698
Another example of the streaming nature of TCP is on the client side. Sometimes, it prints:
-----------------------
Server:
goodbye
Server:
-----------------------goodbye
Server:
Server:
-----------------------
Server:
goodbye
The separate lines in the server:
conn.sendall(liners.encode("utf-8"))
conn.sendall(b'goodbye')
Sometimes get received by the single recv in the client:
server_data = kulmiye.recv(1024).decode("utf-8")
Again, this is due to the streaming nature of TCP and no message boundaries. If you sent each line terminated by a newline, you could call recv() a number of times until a newline was detected, then process just that line.
Another answer that illustrates this technique is here: https://stackoverflow.com/a/55186805/235698.

Thank you Mark Tolonen for your guidance. I still need to brush up how to handle TCP data stream. Nevertheless, I was able to achieve more desirable results using numerous
conn.sendall within my nested for loop. Happy camper!

Socket Programming; File corrupts when transferring over multiple devices

The problem I'm having is to get a file from the server to client across devices. Everything works fine on localhost.
Lets say I want to "get ./testing.pdf" which sends the pdf from the server to the client. It sends but it is always missing bytes. Is there any problems with how I am sending the data. If so how can I fix it? I left out the code for my other functionalities since they are not used for this function.
sending a txt file with "hello" in it works perfectly
server.py
import socket, os, subprocess # Import socket module
s = socket.socket() # Create a socket object
host = socket.gethostname() # Get local machine name
#host = ''
port = 5000 # Reserve a port for your service.
bufsize = 4096
s.bind((host, port)) # Bind to the port
s.listen(5) # Now wait for client connection.
while True:
c, addr = s.accept() # Establish connection with client.
print 'Got connection from', addr
while True:
userInput = c.recv(1024)
.... CODE ABOUT OTHER FUNCTIONALITY
elif userInput.split(" ")[0] == "get":
print "inputed get"
somefile = userInput.split(" ")[1]
size = os.stat(somefile).st_size
print size
c.send(str(size))
bytes = open(somefile).read()
c.send(bytes)
print c.recv(1024)
c.close()
client.py
import socket, os # Import socket module
s = socket.socket() # Create a socket object
host = socket.gethostname() # Get local machine name
#host = '192.168.0.18'
port = 5000 # Reserve a port for your service.
bufsize = 1
s.connect((host, port))
print s.recv(1024)
print "Welcome to the server :)"
while 1 < 2:
userInput = raw_input()
.... CODE ABOUT OTHER FUNCTIONALITY
elif userInput.split(" ")[0] == "get":
print "inputed get"
s.send(userInput)
fName = os.path.basename(userInput.split(" ")[1])
myfile = open(fName, 'w')
size = s.recv(1024)
size = int(size)
data = ""
while True:
data += s.recv(bufsize)
size -= bufsize
if size < 0: break
print 'writing file .... %d' % size
myfile = open('Testing.pdf', 'w')
myfile.write(data)
myfile.close()
s.send('success')
s.close

I can see two problems right away. I don't know if these are the problems you are having, but they are problems. Both of them relate to the fact that TCP is a byte stream, not a packet stream. That is, recv calls do not necessarily match one-for-one with the send calls.
size = s.recv(1024) It is possible that this recv could return only some of the size digits. It is also possible that this recv could return all of the size digits plus some of the data. I'll leave it for you to fix this case.
data += s.recv(bufsize) / size -= bufsize There is no guarantee that that the recv call returns bufsize bytes. It may return a buffer much smaller than bufsize. The fix for this case is simple: datum = s.recv(bufsize) / size -= len(datum) / data += datum.