I need to send big fixed sized buffers from python to c/c++ code side.
From python side I have buffers in form of:
array = np.zeros([1000], dtype=np.uint8)
array = fill(array)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('localhost', 1010))
Code below is used to recv data from connected socket from C/C++ side.From the c/c++ side I need to have this buffer as raw 1000 byte array, I don't need any headers, I need just data. Unfortunately I haven't found howto do it. In general np.tofile()- makes things I need but in file.
int recvBuffer(uint8_t *p_buff, uint32_t size, int socket)
{
uint32_t recived = 0;
uint8_t *p_curr = p_buff;
while (recived < size)
{
int ret = recv(tcp_sock, p_curr, size - recived, 0);
if (ret != -1)
{
p_curr += ret;
recived += ret;
}
else
{
return false;
}
}
return true;
}
I found solution.
I found it in np.tofile description.
np_array = np.zeros([1000], dtype=np.uint8)
np_array = fill(np_array)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('localhost', 1010))
raw_buffer = np_array.to_bytes()
s.send(raw_buffer)
Makes exactlty what I wanted. raw_buffer - holds np.array buffer as raw c-stype buffer.
Related
I am sending .jpg images from Qt client to python server. Qt client sends as follows:
void Sender::sendToServer(QByteArray imageData)
{
QDataStream stream(serverSocket);
int t = imageData.size();
stream << t;
stream << imageData;
serverSocket->waitForBytesWritten(-1);
qDebug() << "Frame size:" << imageData.size();
}
And the server is:
unpacker = struct.Struct('!i')
conn, addr = s.accept()
bool = True
data = b''
while True:
while len(data) < 4:
try:
data += conn.recv(100)
except:
break
img_size = unpacker.unpack(data[:4])[0]
print('size: ', img_size)
print('Data: ',data)
print('Image: ',data[:8]) # deleting preceding 8 bytes
# image processing
The output is:
//Client:
Frame size: 49993
//Server:
size: 49993
Data: b'\x00\x00\xc3\r\x00\x00\xc3\r\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01...
Image: b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00...
In the output, .jpg file starts from '\xff\xd8'. As C++ int is 4 byte, the server expects "{4 byte}\xff\xd8...". However, preceding bytes are always of different size. Sometimes it's 8 bytes, but sometimes even different (such as 6 or 9 bytes). I could not figure out the reason behind it. What is the problem here?
Platform: win7, Qt 5.9.1, Python 3.4.
Edit
Following #Frank's answer, I changed the code as follows and everything works fine:
Qt client send method:
void Sender::sendToServer(QByteArray imageData)
{
QDataStream stream(serverSocket);
QString size = QString::number(data.size());
QString size_8_byte = size.rightJustified(8, ' '); // add spaces to make it exactly 8 bytes
imageData.prepend(size_8_byte.toUtf8());
stream.writeRawData((char*)(imageData.data()), imageData.size());
serverSocket->waitForBytesWritten(-1);
}
Python server:
unpacker = struct.Struct('8s')
conn, addr = s.accept()
data = b''
bool = True
while True:
while len(data) < 8:
try:
data += conn.recv(100)
except:
break
img_size = unpacker.unpack(data[:8])[0]
img_size = int(img_size.decode("utf-8").strip()) #remove spaces and convert to int
data = data[8:] #correct data
#image processing
QDataStream is not a simple binary dumper, it formats the data in some internally defined way. This helps with stuff like endianness correction and the like,
However, there is a very simple fix available to you: QDataStream provides the writeRawData() function, which bypasses all formatting operations.
Simply change your output operations to using it and you should be good to go.
I am sending a double value from C through TCP using lwIP. To accomplish that, I am using an union between a double and one string of 8 bytes (because the lwIP's function to send data is of type 'char *'). Here te code:
void process_echo_request(void *p) {
int sd = (int)p;
int RECV_BUF_SIZE = 2048;
char recv_buf[RECV_BUF_SIZE];
int n, nwrote;
union {
double dval;
char sval[sizeof(double)];
} data_to_send;
data_to_send.dval = 1e+23;
while (1) {
/* read a max of RECV_BUF_SIZE bytes from socket */
if ((n = read(sd, recv_buf, RECV_BUF_SIZE)) < 0) {
xil_printf("%s: error reading from socket %d, closing socket\r\n", __FUNCTION__, sd);
break;
}
/* break if the recved message = "quit" */
if (!strncmp(recv_buf, "quit", 4))
break;
/* break if client closed connection */
if (n <= 0)
break;
data_to_send.dval += 1e+23;
/* handle request */
if ((nwrote = write(sd, data_to_send.sval, 8)) < 0) {
xil_printf("%s: ERROR responding to client echo request. received = %d, written = %d\r\n",
__FUNCTION__, n, nwrote);
xil_printf("Closing socket %d\r\n", sd);
break;
}
}
/* close connection */
close(sd);
vTaskDelete(NULL);
I am trying to read from a Python script, but I fail on get the double value. The code:
#!/usr/bin/env python
import socket
from ctypes import *
TCP_IP = '192.168.0.10'
TCP_PORT = 1000
BUFFER_SIZE = 8
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((TCP_IP, TCP_PORT))
s.send(" ")
data = s.recv(BUFFER_SIZE)
s.close()
print float(data)
It fails with the message ValueError: could not convert string to float: �J��-�D
I know, is because the string is not a literal representation of a real value. Any idea? I need the real (double) value.
For more information, the lwIP echo server is running in a Zynq 7000 FPGA+SoC. I am running the Python code from a Debian 8. The double value is a 64 bit temperature for a sensor.
-- EDIT --
The final working code is:
#!/usr/bin/env python
import socket
from struct import *
TCP_IP = '192.168.0.10'
TCP_PORT = 1000
BUFFER_SIZE = 8
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((TCP_IP, TCP_PORT))
s.send(" ")
data = s.recv(BUFFER_SIZE)
s.close()
print unpack("d",data)[0]
you're recieving the binary representation of the double, not the string one.
You have to unpack it using struct module and the double specifier which is d. struct.unpack returns a tuple, so take first & only value to get your float
my_value = struct.unpack("d",data)[0]
note that you can even fix the endianness with < or >, not necessary if you're on the same machine or CPU type for both programs.
I wrote a tcp server and tcp client, and I want to send a structure from client to server, and then the server send the structure back. Server is written in cpp, and client is written in python.
Then I met a problem, that my structure has a member which type is string, then I cannot find a proper format string to describe this member in python(https://docs.python.org/2/library/struct.html).
So I want to know how can I describe the string attribute in python?
below is the minimal code:
------server---------
#include <stdlib.h>
#include <unistd.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <cstring>
#include <iostream>
struct Message
{
int id;
std::string text;
};
int main()
{
int sockfd = socket(AF_INET, SOCK_STREAM, 0);
struct sockaddr_in my_addr;
bzero(&my_addr, sizeof(my_addr));
my_addr.sin_family = AF_INET;
my_addr.sin_port = htons(11111);
my_addr.sin_addr.s_addr = htonl(INADDR_ANY);
unsigned retry_times = 0;
while(retry_times < 3)
{
if(bind(sockfd, (struct sockaddr*)&my_addr, sizeof(my_addr)) == 0)
{
break;
}
sleep(1000);
++retry_times;
}
listen(sockfd, 1);
struct sockaddr_in client_addr;
socklen_t addr_len = sizeof(client_addr);
int connfd = accept(sockfd, (struct sockaddr*)&client_addr, &addr_len);
while(true)
{
Message msg;
unsigned int len = recv(connfd, &msg, sizeof(msg), 0);
if(len == 0)
{
break;
}
std::cout << msg.id << "--" << msg.text << std::endl;
send(connfd, &msg, sizeof(msg), 0);
}
close(connfd);
close(sockfd);
}
-----------client-----------
import socket
import struct
class TCPClient:
def __init__(self):
self.server_address = ('127.0.0.1', 11111)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect_ex(self.server_address)
def send(self, msg):
self.sock.send(msg)
def recv(self):
return self.sock.recv(512)
def close(self):
self.sock.close()
if __name__ == '__main__':
client = TCPClient()
fmt = 'is' #### what format string will be better? this fmt is useless
msg = bytearray(512)
struct.pack_into(fmt, msg, 0, 1, "Hello")
client.send(msg)
data = client.recv()
print struct.unpack_from(fmt, data)
client.close()
There is no Python format string which will fix this problem. The root cause is that you are sending the bytes of a std::string on the network. But a std::string often contains a pointer (and size) to the actual content, which means you are not sending the actual content at all!
To fix it, you have two main choices--either modify your struct:
struct Message
{
int32_t id; // use a fixed-size integer type
char text [50]; // assumes you know the max length
};
Or more conventionally, write a serialization routine:
void send_msg(int fd, const Message& msg)
{
// XXX: you must check the return value of send()
send(fd, &msg.id, sizeof(msg.id), 0);
uint32_t text_size = msg.text.size();
send(fd, &text_size, sizeof(text_size), 0);
send(fd, &msg.text.data(), msg.text.size(), 0);
}
Now we are sending the ID (4 bytes, Python format i), the text_size (4 bytes, unsigned, Python format I), then the text (text_size bytes, Python format '{}s'.format(text_size)). You can unpack it in two steps on the receiving end: first unpack iI to get the ID and text_size, then unpack '{}s'.format(text_size) to get the text.
Note that since you are using TCP, partial writes and reads are possible. So you need to deal with the fact that your sender might send only half a message, and your receiver might receive only half a message. Your current logic does not handle this (but will likely appear to work most of the time if text_size is less than 500).
My (Python) publisher:
import zmq
import time
context = zmq.Context()
socket = context.socket(zmq.PUB)
connectStr = "tcp://*:%d" % 5563
socket.bind(connectStr)
messageNum = 0
while True:
++messageNum
message = "Testing %d"%messageNum
print("Sending.. '%s'"%message)
socket.send_string(message)
time.sleep(1)
messageNum += 1
My (C++) subscriber (running in GTest):
TEST(ZeroMqPubSubTest, SubscribeGetsData)
{
// Set up the subscriber we'll use to receive the message.
zmq::context_t context;
zmq::socket_t subscriber(context, ZMQ_SUB);
// Connect to the publisher
subscriber.connect("tcp://127.0.0.1:5563");
subscriber.setsockopt(ZMQ_SUBSCRIBE, ""); // Set the filter blank so we receive everything
zmq::message_t response(0);
EXPECT_TRUE(subscriber.recv(&response));
}
I start up the publisher then start up the subscriber. The latter never returns though.
If I run a Python subscriber doing (I thought) exactly the same thing..
import zmq
context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.connect ("tcp://127.0.0.1:5563")
socket.setsockopt_string(zmq.SUBSCRIBE, "")
print ("Waiting for data...")
while True:
message = socket.recv()
print ("Got some data:",message)
..it works fine:
Waiting for data...
Got some data: b'Testing 8'
Got some data: b'Testing 9'
There are two overloads of setsockopt defined in zmq.hpp:
template<typename T> void setsockopt(int option_, T const& optval)
{
setsockopt(option_, &optval, sizeof(T) );
}
inline void setsockopt (int option_, const void *optval_, size_t optvallen_)
{
int rc = zmq_setsockopt (ptr, option_, optval_, optvallen_);
if (rc != 0)
throw error_t ();
}
By providing only two arguments you implicity used the first overload, which assumes a value length of sizeof(T). This resolves to one, because "" is a zero-terminated character array. To pass in an empty string you need to use the second overload and specify a length of 0:
subscriber.setsockopt(ZMQ_SUBSCRIBE, "", 0);
Alternatively, use a zero size data type:
char none[0];
subscriber.setsockopt(ZMQ_SUBSCRIBE, none);
I would like to set up a serial communication between a Python daemon and an Arduino.
At first, the Python daemon sets up a serial connection that will last for the whole lifetime of the daemon. Through this connection, I would like to send data to the Arduino and receive back data in the acks variable every time the Python daemon receives commands.
The problem is that while the first time the communication goes well, nothing is sent through serial afterwards. If I make the a new connection for every request it works, but it makes the program very slow, which I'd like to avoid.
edit: the real issue is when send a correct string to the arduio evrything goes well but when i send a wrong one the serial port block and it will never reconize corrct strings again( the problem is in the arduino code)
Python code:
import serial
import time
import sys
from socket import *
import threading
import thread
def handler(clientsock,addr):
while 1:
#arduino.flush()
data = clientsock.recv(BUFSIZ)
if not data:
break
print data
print data
#time.sleep(3)
arduino.write(data)
#time.sleep(3)
ack = arduino.readline(1)
arduino.flush()
clientsock.send(ack+"\n")
clientsock.close()
if __name__=='__main__':
HOST = '0.0.0.0'
PORT = 21567
BUFSIZ = 1024
ADDR = (HOST, PORT)
arduino = serial.Serial('/dev/ttyACM0',9600,timeout=6)
serversock = socket(AF_INET, SOCK_STREAM)
serversock.bind(ADDR)
serversock.listen(2)
while 1:
print 'waiting for connection...'
clientsock, addr = serversock.accept()
print '...connected from:', addr
thread.start_new_thread(handler, (clientsock, addr))
Arduino code:
int relayPinCH1 = 7; // pin de commande du relais 1
char inData[20]; // Allocate some space for the string
char inChar=-1; // Where to store the character read
byte index = 0; // Index into array; where to store the character
void setup()
{
pinMode(relayPinCH1, OUTPUT);
Serial.begin(9600);
}
char Comp(char* This) {
while (Serial.available() > 0) // Don't read unless
// there you know there is data
{
if(index < 19) // One less than the size of the array
{
inChar = Serial.read(); // Read a character
inData[index] = inChar; // Store it
index++; // Increment where to write next
inData[index] = '\0'; // Null terminate the string
}
}
Serial.flush();
if (strcmp(inData,This) == 0) {
for (int i=0;i<19;i++) {
inData[i]=0;
}
index=0;
return(0);
}
else {
return(1);
}
}
void loop()
{
//Serial.println("Hello Pi");
if (Comp("l11\n")==0)
{
Serial.flush();
digitalWrite(relayPinCH1, HIGH);
Serial.println("y");
}
if (Comp("l10\n")==0)
{
Serial.flush();
digitalWrite(relayPinCH1, LOW);
Serial.println("n");
}
delay(1000);
}
In your Arduino code, your logic is kind of funky - so, I'm not sure, but are you clearing index to 0 before you start the loop again? It looks like once index == 19, it may or may not get reset to 0 depending upon later logic. If you enter Comp() a second time and index >= 19 then you'll never read the serial port again.
I think #Zeus is entirely right (and hence I upvoted that answer), but there are also other problems. To reiterate what #Zeus is saying:
index is only reset to 0 if the comparison succeeds. So your buffer is full, the string you are looking for isn't there, and index never goes back to 0 again.
Once index reaches 19, no more reading is done. As a result, whatever is in inData stays in inData and all the future comparisons will fail, meaning index will never get reset to 0.
There are a number of other problems in the code, but the main issue is that the design is very fragile, and prone to exactly the sort of error you are experiencing. For instance if the newlinews your Python script is sending are CR+LF for newlines, but you are expecting CR only, you'll have the same sort of failure you have now: first time communications work, but never again.
I would suggest reorganizing your code like this:
Your function for reading serial port reads a line from a serial port and returns that to the caller (without the newlines), regardless of the content of the communications.
The caller compares the line received from the serial port with the list of known commands and executes them accordingly.
This might look rougly as follows
char strCommand[0xFF];
int idxCommandChar;
// Read a command from serial, returning the command size
// This function BLOCKS, i.e., doesn't return until a command is available
int readSerialCommand() {
// We reset the index to zero on every read: the command is overwritten every time
idxCommandChar = 0;
// Read serial characters and store them in strCommand
// until we get a newline
int in = Serial.read();
while (in!='\n') {
strCommand[idxCommandChar++] = in;
in = Serial.read();
}
// Add the string terminator
strCommand[idxCommandChar++] = '\0';
// Return command size
return idxCommandChar;
}
// Get command from serial, and process it.
void processCommand() {
readSerialCommand();
if (strcmp(strCommand, "CMD1")==0) {
// do something
} else if (strcmp(strCommand, "CMD2")==0) {
// do something else
} else {
// Unknown command
Serial.println("Unknown command");
}
}
void loop() {
processCommand();
delay(1000);
}
This code blocks on serial, i.e. doesn't return until a newline is detected. You could easily modify the code to be non-blocking, perhaps like this:
/* Read serial characters, if available and store them in strCommand
until we get a newline
Returns 0 if no command is available */
int readSerialCommand() {
idxCommandChar = 0;
while (Serial.available()) {
int in = Serial.read();
while (in!='\n') {
strCommand[idxCommandChar++] = in;
in = Serial.read();
}
strCommand[idxCommandChar++] = '\0';
return idxCommandChar;
}
return 0;
}
// Get command from serial (if available), and process it.
void processCommand() {
if (readSerialCommand()) {
....
In either case you might loose serial characters while you are waiting, so you may want to rethink that strategy.