I am sending 32 bytes packets every 1ms to this socket. I wish to print the data after every 40 ms. And apparently the code does that. But even when I stop sending data, I still continue to see that data is being printed.
Is it holding the data in some cache? or simply the python socket has a huge delay? Why?
The code is as follows:
## Import necessary libraries
import math
import numpy as np
import socket
import struct
import time
from synchrophasor.frame import CommandFrame
from datetime import datetime
## Configure socket for Phasor data ##
UDP_IP = "10.10.114.22"
UDP_PORT = 8208 #UDP phasor values 32 bytes (V,phi,P)
sock_ph = socket.socket(socket.AF_INET, # Internet
socket.SOCK_DGRAM) # UDP
sock_ph.bind((UDP_IP, UDP_PORT))
print("socket bound, waiting for data...")
while True:
raw = sock_ph.recv(32)
#print(raw)
mag = struct.unpack('d', raw[8:16])[0]
# print("mag =",mag,type(mag))
angle = struct.unpack('d', raw[16:24])[0]
# print("angle =",angle,type(angle))
header = struct.unpack('d', raw[0:8])[0]
# print("header =",header,type(header))
phasor = (mag, angle)
Vol_A=raw
VA = float(mag)
phi_A = float(angle)
VB = VA
phi_B = phi_A+(math.pi) * 2 / 3
VC = VA
phi_C = phi_A-(math.pi) * 2 / 3
time.sleep(1/25)
# pmu.send_data(phasors=[(VA,phi_A),(VB,phi_B),(VC,phi_C)],analog=[9.91],digital=[0x0001])
#time.sleep(1/config_rr)
print([(VA,phi_A),(VB,phi_B),(VC,phi_C),datetime.now()])
most programs don't want to discard unread datagrams so most OSs will buffer them for you. your case is somewhat unusual so you'd need to write code to handle this case. I'd change your code to do something like:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.bind(('', 8208))
# block until we read an initial packet
raw = s.recv(1024)
s.setblocking(False)
while True:
# unpack
header, mag, angle = struct.unpack('ddd', raw)
# do something with data
print(f'header={header} mag={mag} angle={angle}')
# sleep for some time
time.sleep(1/25)
# discard any packets you've received in the mean time
while True:
try:
raw = s.recv(1024)
except OSError as err:
# OS buffer is empty: we've therefore got the most recent data
if err.errno == socket.EWOULDBLOCK:
break
# something else failing, reraise the error
raise
note that Steffen Ullrich's suggestion of sending the data at the correct rate would be easier, but assumes that you have control over the sending process. the fact that you said "I am sending" suggests you do, and so would likely make a better solution
Related
I'm using the socket module for a UDP server。The incoming packets always have a different size(0-65535), so client send package length first, then send the package；server receives data according to the package length and 1024 per server receives。but server didn't work like I thought，it can receives package only once。
# -*- coding: utf-8 -*-
import json
import socket
ip_port = ('127.0.0.1', 8080)
server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server.bind(ip_port)
print('server listen......')
def recv_header(s_socket):
try:
msg, addr = s_socket.recvfrom(32)
return json.loads(msg), addr
except Exception as e:
return str(e), None
while True:
header_msg, client_addr = recv_header(s_socket=server)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
receiverBufsize = 1024
if client_addr is None:
print (header_msg)
continue
data_body = ""
package_len = int(header_msg['length'])
print ("package_len: {}".format(package_len))
print ("client_addr: {}".format(str(client_addr)))
while package_len > 0:
if package_len > receiverBufsize:
body_part = server.recvfrom(receiverBufsize)
else:
body_part = server.recvfrom(package_len)
data_body += body_part[0]
package_len -= receiverBufsize
print ("data_body: {}".format(data_body))
# -*- coding: utf-8 -*-
import json
import socket
import random
ip_port = ('127.0.0.1', 8080)
client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def get_data(length):
base_str = 'ABCDEFGHIGKLMNOPQRSTUVWXYZabcdefghigklmnopqrstuvwxyz0123456789'
return ''.join([random.choice(base_str) for _ in range(length)])
while 1:
msg = raw_input('>>>：')
if msg == 'q':
break
if not msg:
continue
len = int(msg)
client.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
data = get_data(len)
header = json.dumps({"length": len})
aa = client.sendto(header.encode('utf-8'), ip_port)
print aa
aa = client.sendto(data.encode('utf-8'), ip_port)
print aa
If the packet length is less than 1024(my receiverBufsize), it works normally
enter image description here
enter image description here
If the packet length is more than 1024(my receiverBufsize), socket.recvfrom will block
enter image description here
enter image description here
UDP is a message oriented protocol, not a data stream like TCP. This means in UDP a single send is matched by a single recv.
This means you cannot use multiple recv with a small buffer (size 1024 in your case) to get data which were sent in a single send with a larger buffer - you need to use recv with a large enough buffer instead.
This means also that you don't actually need to send the length upfront, since you don't need to add message semantics to a protocol which already has message semantic. In contrary, sending the length in a separate message as you do could actually be harmful since UDP is an unreliable protocol and you cannot rely on packets getting delivered in order, at all or only once.
I've been sketching some test code for a file transfer using a TCP socket in python (3.9 - Win10 x64). When I run the code below the NIC throughput in task manager increases by about 100Mb/s (goes back down when the socket is broken). The data rates on the hard drives and the measured rate in the code seem to indicate that the actual transfer rate of the intended data is about 11Mb/s. First the code and next some more info about what I have been trying:
Client
import socket, os, sys
from threading import Thread
from time import monotonic
file_send = r'PATH TO FILE'
stats = os.stat(file_send)
print(stats.st_size)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as client_soc:
client_soc.connect(('192.168.1.88', 6325))
client_soc.sendall(str(stats.st_size).encode('utf-8'))
client_soc.recv(10)
buffer_size = 1024 * 32
#client_soc.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, buffer_size)
with open(file_send, 'rb') as f:
bytes_read = 0
s_time = monotonic()
check_in = s_time
while bytes_read < stats.st_size:
read = f.read(1024*1024*10)
mono = monotonic()
t_diff = mono - s_time
if mono - check_in > 10:
print('{:,.0f} Kb/s'.format((bytes_read / 1024) / t_diff))
check_in = mono
client_soc.send(read)
bytes_read += len(read)
print('Done sending')
Server
import socket
from threading import Thread
class DLThread(Thread):
def __init__(self, bind_address, port, file_recv):
super(DLThread, self).__init__()
self.life_switch = True
self.bind_address = bind_address
self.port = port
self.file_recv = file_recv
def run(self) -> None:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listen_doc:
listen_doc.bind((self.bind_address, self.port))
listen_doc.listen()
print('listening')
client_conn, addr = listen_doc.accept()
with client_conn:
print('Connected')
data = client_conn.recv(1024)
size = int(data)
print('File size: {}'.format(size))
client_conn.sendall(b'k')
gotten = 0
percent_inc = 0.1
percent_chunk = percent_inc * size
thresh = percent_inc * size
with open(self.file_recv, 'wb') as f:
while gotten < size:
data = client_conn.recv(1024*1024*5)
f.write(data)
gotten += len(data)
if gotten > thresh:
print('{:.0%} Complete'.format(gotten / size))
thresh = int(gotten // percent_chunk) + percent_chunk
def pull_the_plug(self):
self.life_switch = False
self.join()
bind_addr = input('Bind Address: ')
port = int(input('port: '))
file_location = input('fileloc: ')
dl = DLThread(bind_addr, port, file_location)
dl.start()
print('Started')
dl.join()
I tried changing the buffer sizes ad using send / sendall. I also tried to have the client wait for a (redundant - I think) acknowledgement from the server at each chunk.
I took a look at the traffic in wireshark and I am seeing messages that appear to be padded with a bunch of 0s appended. I am not sure if this is a clue or not.
I can't think of why there would be so much overhead on the network interface. It feels weird just throwing all the data into the socket and letting the API do everything else but this is what many other people have done in their examples.
Sometimes with posts like this people recommend using a different tool. I just want to make it clear that I am not really trying to get this file transfer to work, I just want to understand why this is happening so I can learn something.
Although this is a very specific example of a common mistake, I'll confirm the answer here. Joachim Isaksson is correct. Task Manager reports network traffic in orders of bits/second not bytes/second. This is almost an order of magnitude different.
what learn? Pay attention to units and when things don't line up look for how the data could be related.
I would like to read one by one the objects coming from a TCP stream, preferably using the MessagePack library.
On one side I have a client that, at each iteration of a loop:
computes a point location (tuple)
packs that location and sends it through a socket
On the other side, a server that:
receive the data when client is detected
unpack that data
For now I am storing the data in a buffer on reception, then proceed to unpacking when the stream is over. My problem is that I need to unpack the tuples one by one as they are sent. In other words I would like to read the data in real time without putting it in a buffer.
Provided it is possible, how could I achieve this using MessagePack ?
-- client side --
#Python3.7
import socket
import msgpack
import math
HOST = "127.0.0.1"
PORT = 9000
den = 40
rad = 100
theta = math.tau / den
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.connect((HOST, PORT)) #connect to server
for step in range(den):
x = math.cos(i*theta) * rad
y = math.sin(i*theta) * rad
data = msgpack.packb((x, y), use_bin_type = True)
sock.sendall(data)
-- server side --
#Jython2.7 <-- Python 2.7 compatible only
from io import BytesIO
import msgpack
import socket
HOST = "127.0.0.1"
PORT = 9000
buf = BytesIO()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(1)
connection, address = s.accept()
while True:
try:
data = connection.recv(1024)
buf.write(data)
except:
buf.seek(0)
unpacker = msgpack.Unpacker(buf, use_list=False, raw=False)
for unpacked in unpacker:
print(unpacked)
buf = BytesIO()
See "Stream Unpacking" section in the README:
https://github.com/msgpack/msgpack-python#streaming-unpacking
You can do like this:
unpacker = msgpack.Unpacker(use_list=False, raw=False)
while True:
data = connection.recv(1024)
if not data:
break
unpacker.feed(data)
for unpacked in unpacker:
print(unpacked)
I would like to send the location of a moving point to a server via TCP with the socket module. That point location is updated at each iteration of a for loop and is sent in the form of a tuple (x, y) that has been serialized with pickle dumps methods.
Problem:
On the server side, it seems that I only get to receive the location from the first iteration of that loop. As if all the following updated positions had been skipped or lost in the process.
I can’t say for sure what is the reason behind this behavior but my bet is that I am not correctly setting things on the server side. I suspect the data to be sent entirely but not processed adequately on reception due to some mistakes that I am probably doing with the socket module (I am completely new to the world of network interfaces).
Code:
--client side--
#Python3.7
import socket
import pickle
import math
HOST = "127.0.0.1"
PORT = 12000
den = 20
rad = 100
theta = math.tau / den
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.connect((HOST, PORT)) #connect to server
for step in range(1000):
i = step%den
x = math.cos(i*theta) * rad
y = math.sin(i*theta) * rad
data = pickle.dumps((x, y), protocol=0)
sock.sendall(data)
--server side--
#Jython2.7
import pickle
import socket
HOST = "127.0.0.1"
PORT = 12000
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(1)
while True:
connection, address = s.accept()
if connection:
data = connection.recv(4096)
print(pickle.loads(data)) # <-- only print once (first location)
You need to put connection, address = s.accept() outside the while loop otherwise your server will wait for a new connection every time.
You also have an issue with the way your are receiving data. connection.recv(4096) will return any amount of bytes between 0 and 4096 not every time a complete "data" message is received. To handle this you could send a header before sending you json indicating how much data should be received
By adding a header, you will make sure the data messages you are sending will be received properly.
The header in this example is a four bytes int indicating the size of data.
Server
import pickle
import socket
import struct
HEADER_SIZE = 4
HOST = "127.0.0.1"
PORT = 12000
def receive(nb_bytes, conn):
# Ensure that exactly the desired amount of bytes is received
received = bytearray()
while len(received) < nb_bytes:
received += conn.recv(nb_bytes - len(received))
return received
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(1)
connection, address = s.accept()
while True:
# receive header
header = receive(HEADER_SIZE, connection)
data_size = struct.unpack(">i", header)[0]
# receive data
data = receive(data_size, connection)
print(pickle.loads(data))
Client
import socket
import pickle
import math
HEADER_SIZE = 4
HOST = "127.0.0.1"
PORT = 12000
den = 20
rad = 100
theta = math.tau / den
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.connect((HOST, PORT)) #connect to server
for step in range(1000):
i = step%den
x = math.cos(i*theta) * rad
y = math.sin(i*theta) * rad
data = pickle.dumps((x, y), protocol=0)
# compute header by taking the byte representation of the int
header = len(data).to_bytes(HEADER_SIZE, byteorder ='big')
sock.sendall(header + data)
Hope it helps
Python27. I am trying to stream the video from camera between client and server apps.
I am learning python from couple of days... I have already learned that to send the data over UDP I need to serialize the data... I have found the pickle library but the code does not work for me. Is there something more regarding data serialization to be able to send the data over the network? How to divide that into smaller parts of data? When i check the type it says that it is string with the size of about 1MB or even more. I guess that's why I'm getting the error:
socket.error: [Errno 10040] A message sent on a datagram socket was larger than the internal message buffer or some other network limit, or the buffer used to receive a datagram into was smaller than the datagram itself
TCP version of this worked fine, but performance was poor. I wanted to try with UDP
Here's my code:
__AUTHOR__=u"DK"
import cv2
import numpy as np
import socket
import sys
import pickle
import struct
def StartStreamSending():
u"""
Funkcja przesyłająca stream z lokalnej kamery[0] na serwer.
"""
UDP_IP = "127.0.0.1"
UDP_PORT = 8012
cap=cv2.VideoCapture(0)
clientsocket=socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
while True:
ret,frame=cap.read()
#result, img_str = cv2.imencode('.jpeg', frame)
#img_str2 = np.array(img_str)
#pik = pickle.Pickler()
#frame = np.reshape(frame, (240,320,3))
data = np.array(frame)
dataToSend = pickle.dumps(data)
size = sys.getsizeof(dataToSend)
print(size)
print(type(dataToSend));
clientsocket.sendto(dataToSend, (UDP_IP, UDP_PORT))
StartStreamSending()
And the server:
__AUTHOR__=u"DK"
import socket
import sys
import cv2
import pickle
import numpy as np
import struct
HOST='localhost'
PORT=8012
def WaitAndReceiveVideoStream(_strHOST, _iPORT):
u"""
Funkcja odbierająca stream z kamery klienta.
"""
s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
print 'Socket created'
s.bind((_strHOST,_iPORT))
print 'Socket bind complete'
data = ""
#payload_size = struct.calcsize("L")
i = 0
while True:
data, addr = s.recvfrom(512)
frame=pickle.loads(data)
i = i + 1
print 'coming frame' + str(i)
#frame = numpy.fromstring(data, dtype=numpy.uint8)
#frame = numpy.reshape(frame, (240,320,3))
#if 3 == i :
# cv2.SaveImage("C:\\image.",frame)
cv2.imshow('frame',frame)
cv2.waitKey(4) #Important delay - for test try 2 or 4 value.
s.close()
WaitAndReceiveVideoStream(HOST, PORT)