Develop Reference

Python bytes buffer only giving out zeros even though data is being correctly received through SPI on Raspberry Pi

This is a follow up question from: <class 'TypeError'>: LP_c_long instance instead of list but is a different issue so I am posting a new question here.
I am using ctypes to invoke a c function from python script. The c function which is being invoked is:
uint32_t crc32Word(uint32_t crc, const void *buffer, uint32_t size)
I have this python code:
import datetime
import os
import struct
import time
import pigpio
import spidev
import ctypes
lib = ctypes.CDLL('/home/pi/serial_communication/crc.so')
lib.crc32Word.argtypes = ctypes.c_uint32, ctypes.c_void_p, ctypes.c_uint32
lib.crc32Word.restype = ctypes.c_uint32
bus = 0
device = 0
spi = spidev.SpiDev()
spi.open(bus, device)
spi.max_speed_hz = 4000000
spi.mode = 0
pi.set_mode(12, pigpio.INPUT)
C=0
def output_file_path():
return os.path.join(os.path.dirname(__file__),
datetime.datetime.now().strftime("%dT%H.%M.%S") + ".csv")
def spi_process(gpio,level,tick):
print("Detected")
data = bytes([0]*1024)
spi.xfer([0x02])
with open(output_file_path(), 'w') as f:
t1=datetime.datetime.now()
for x in range(1):
spi.xfer2(data)
values = struct.unpack("<" +"I"*256, bytes(data))
C = lib.crc32Word(0xffffffff,data,len(data))
f.write("\n")
f.write("\n".join([str(x) for x in values]))
t2=datetime.datetime.now()
print(t2-t1)
print(C)
input("Press Enter to start the process ")
spi.xfer2([0x01])
cb1=pi.callback(INTERRUPT_GPIO, pigpio.RISING_EDGE, spi_process)
while True:
time.sleep(1)
Previously, I initialized data as data = [0]*1024, so I was receiving some error as mentioned in the previous post. But the error was resolved by initializing data to bytes. Now the issue that I have is even though the correct data is received on Raspberry Pi (checked using logic analyzer), only 0s are stored in the file.
I also tried initializing data as arrays instead of lists using numpy library as below, but I receive an error saying:
Tried code:
import numpy as np
#changed initialization of data to:
data= np.arange(1024) #also tried: np.zeros(1024,ctypes.c_uint8), also get same error
Error:
File "2crc_spi.py", line 48, in spi_process
spi.xfer2(data)
TypeError: Non-Int/Long value in arguments: b5410b00
But data only receives a byte at a time so not sure where is the issue when using array.
Can someone please help? Thanks~
EDIT:
Below is the original code that was working fine with the data collection and unpacking functions before I started integrating the CRC functionality:
import datetime
import os
import struct
import time
import pigpio
import spidev
bus = 0
device = 0
spi = spidev.SpiDev()
spi.open(bus, device)
spi.max_speed_hz = 4000000
spi.mode = 0
pi.set_mode(12, pigpio.INPUT)
a=0
def output_file_path():
return os.path.join(os.path.dirname(__file__),
datetime.datetime.now().strftime("%dT%H.%M.%S") + ".csv")
def spi_process(gpio,level,tick):
print("Detected")
data = [0]*2048
spi.xfer([0x02])
with open(output_file_path(), 'w') as f:
t1=datetime.datetime.now()
for x in range(1):
spi.xfer2(data)
values = struct.unpack("<" +"I"*256, bytes(data))
f.write("\n")
f.write("\n".join([str(x) for x in values]))
t2=datetime.datetime.now()
print(t2-t1)
input("Press Enter to start the process ")
spi.xfer2([0x01])
cb1=pi.callback(INTERRUPT_GPIO, pigpio.RISING_EDGE, spi_process)
while True:
time.sleep(1)

As #Mark's suggested in comments, you would have to store the data into another variable/list, for example: recv = spi.xfer2(data). Then you would need to use this recv in your unpack function.
Further, you could also use a python library called zlib instead of using a c library (there are other python libraries as well).
Another point, since zlib takes only bytes as input, you would need to convert recv into bytes (here recv is a list; check my code).
I modified some of your code.
import datetime
import os
import struct
import time
import pigpio
import spidev
import zlib
bus = 0
device = 0
spi = spidev.SpiDev()
spi.open(bus, device)
spi.max_speed_hz = 4000000
spi.mode = 0
pi.set_mode(12, pigpio.INPUT)
C=0
def output_file_path():
return os.path.join(os.path.dirname(__file__),
datetime.datetime.now().strftime("%dT%H.%M.%S") + ".csv")
def spi_process(gpio,level,tick):
print("Detected")
data = bytes([0]*1024)
spi.xfer2([0x02])
with open(output_file_path(), 'w') as f:
t1=datetime.datetime.now()
for x in range(1):
recv = spi.xfer2(data)
values = struct.unpack("<" +"I"*256, bytes(recv))
C = zlib.crc32(bytes(recv))
f.write("\n")
f.write("\n".join([str(x) for x in values]))
t2=datetime.datetime.now()
print(t2-t1)
print(C)
input("Press Enter to start the process ")
spi.xfer2([0x01])
cb1=pi.callback(INTERRUPT_GPIO, pigpio.RISING_EDGE, spi_process)
while True:
time.sleep(1)

Arduino MLX90614 to Pyserial error converting String to Float

Good Day! I was struggling about my arduino MLX90614 code my pyserial cant read the data it show's
Here's my arduino MLX90614 code:
#include <Wire.h>
#include <Adafruit_MLX90614.h>
Adafruit_MLX90614 mlx = Adafruit_MLX90614();
void setup() {
Serial.begin(9600);
mlx.begin();
}
void loop() {
Serial.print("Ambient = "); Serial.print(mlx.readAmbientTempC());
Serial.print("*C\tObject = "); Serial.print(mlx.readObjectTempC()); Serial.println("*C");
Serial.print("Ambient = "); Serial.print(mlx.readAmbientTempF());
Serial.print("*F\tObject = "); Serial.print(mlx.readObjectTempF()); Serial.println("*F");
Serial.println();
delay(500);
}
And heres my python code "Pyserial" to read the data from my arduino
import csv
from time import time
import serial
# Your serial port might be different!
ser = serial.Serial('COM5', timeout=1)
f = open("df.csv", "a+")
writer = csv.writer(f, delimiter=',')
while True:
s = ser.readline().decode()
if s != "":
rows = [float(x) for x in s.split(',')]
# Insert local time to list's first position
rows.insert(0, int(time()))
print(rows)
writer.writerow(rows)
f.flush()
But in result it show's this
"rows = [float(x) for x in s.split(',')]
ValueError: could not convert string to float: 'Ambient 30c /n'"
Someone please help me with this I am really struggling with connecting my Arduino MLX90614 to my Arduino. Thanks in advance

If s != '' is not a space, add the whitespace between the quotes, this is returning values that cannot be passed to the int constructor.
Comment the list comprehension out and print instead to see the current values your code is storing.

Python - Accelerometer reading and writing to CSV file at 1 kHz rate

I am trying to use a MPU-6000 accelerometer and Raspberry Pi Zero W to log vibration data in a windshield. I'm fairly new to Python so please bear with me.
I've written a python2 script that configures the MPU-6000 to communicate over I2C, with the clock configured to 400 kHz.
The MPU-6000 gives an interrupt when there is new data available in the accelerometer registers, which is read, converted to 2's complement and then written to a CSV file together with a timestamp. The output rate of the accelerometer is configured to be 1 kHz.
I'm experiencing that when sampling all three sensor axis the script isn't able to write all data points to the CSV file. Instead of a 1000 datapoints pr axis pr second I get approximately 650 datapoints pr axis pr second.
I've tried writing only one axis, which proved succesfull with 1000 datapoints pr second. I know that the MPU-6000 has a FIFO register available, which I probably can burst read to get 1000 samples/s without any problem. The problem will be obtaining a timestamp for each sample, so I haven't tried to implement reading from the FIFO register yet.
I will most likely do most of the post processing in Matlab, so the most important things the python script should do is to write sensor data in any form to a CSV file at the determined rate, with a timestamp.
Is there any way to further improve my Python script, so I can sample all three axis and write to a CSV file at a 1 kHz rate?
Parts of my script is depicted below:
#!/usr/bin/python
import smbus
import math
import csv
import time
import sys
import datetime
# Register addresses
power_mgmt_1 = 0x6b
power_mgmt_2 = 0x6c
samlerate_divider = 0x19
accel_config = 0x1C
INT_Enable = 0x38
def read_byte(reg):
return bus.read_byte_data(address, reg)
def read_word(reg):
h = bus.read_byte_data(address, reg)
l = bus.read_byte_data(address, reg+1)
value = (h <<8)+l
return value
def read_word_2c(reg):
val = read_word(reg)
if (val >= 0x8000):
return -((65535 - val) + 1)
else:
return val
csvwriter = None
def csv_open():
csvfile = open('accel-data.csv', 'a')
csvwriter = csv.writer(csvfile)
def csv_write(timedelta, accelerometerx, accelerometery, accelerometerz):
global csvwriter
csvwriter.writerow([timedelta, accelerometerx, accelerometery,
accelerometerz])
# I2C configs
bus = smbus.SMBus(1)
address = 0x69
#Power management configurations
bus.write_byte_data(address, power_mgmt_1, 0)
bus.write_byte_data(address, power_mgmt_2, 0x00)
#Configure sample-rate divider
bus.write_byte_data(address, 0x19, 0x07)
#Configure data ready interrupt:
bus.write_byte_data(address,INT_Enable, 0x01)
#Opening csv file and getting ready for writing
csv_open()
csv_write('Time', 'X_Axis', 'Y_Axis', 'Z_Axis')
print
print "Accelerometer"
print "---------------------"
print "Printing acccelerometer data: "
#starttime = datetime.datetime.now()
while True:
data_interrupt_read = bus.read_byte_data(address, 0x3A)
if data_interrupt_read == 1:
meas_time = datetime.datetime.now()
# delta_time = meas_time - starttime
accelerometer_xout = read_word_2c(0x3b)
accelerometer_yout = read_word_2c(0x3d)
accelerometer_zout = read_word_2c(0x3f)
# accelerometer_xout = read_word(0x3b)
# accelerometer_yout = read_word(0x3d)
# accelerometer_zout = read_word(0x3f)
# accelerometer_xout_scaled = accelerometer_xout / 16384.0
# accelerometer_yout_scaled = accelerometer_yout / 16384.0
# accelerometer_zout_scaled = accelerometer_zout / 16384.0
# csv_write(meas_time, accelerometer_xout_scaled,
accelerometer_yout_scaled, accelerometer_zout_scaled)
csv_write(meas_time, accelerometer_xout, accelerometer_yout,
accelerometer_zout)
continue

If the data you are trying to write is continuous, then the best approach is to minimise the amount of processing needed to write it and to also minimise the amount of data being written. To do this, a good approach would be to write the raw data into a binary formatted file. Each data word would then only require 2 bytes to be written. The datetime object can be converted into a timestamp which would need 4 bytes. So you would use a format such as:
[4 byte timestamp][2 byte x][2 byte y][2 byte z]
Python's struct library can be used to convert multiple variables into a single binary string which can be written to a file. The data appears to be signed, if this is the case, you could try writing the word as is, and then using the libraries built in support for signed values to read it back in later.
For example, the following could be used to write the raw data to a binary file:
#!/usr/bin/python
import smbus
import math
import csv
import time
import sys
import datetime
import struct
# Register addresses
power_mgmt_1 = 0x6b
power_mgmt_2 = 0x6c
samlerate_divider = 0x19
accel_config = 0x1C
INT_Enable = 0x38
def read_byte(reg):
return bus.read_byte_data(address, reg)
def read_word(reg):
h = bus.read_byte_data(address, reg)
l = bus.read_byte_data(address, reg+1)
value = (h <<8)+l
return value
# I2C configs
bus = smbus.SMBus(1)
address = 0x69
#Power management configurations
bus.write_byte_data(address, power_mgmt_1, 0)
bus.write_byte_data(address, power_mgmt_2, 0x00)
#Configure sample-rate divider
bus.write_byte_data(address, 0x19, 0x07)
#Configure data ready interrupt:
bus.write_byte_data(address, INT_Enable, 0x01)
print
print "Accelerometer"
print "---------------------"
print "Printing accelerometer data: "
#starttime = datetime.datetime.now()
bin_format = 'L3H'
with open('accel-data.bin', 'ab') as f_output:
while True:
#data_interrupt_read = bus.read_byte_data(address, 0x3A)
data_interrupt_read = 1
if data_interrupt_read == 1:
meas_time = datetime.datetime.now()
timestamp = time.mktime(meas_time.timetuple())
accelerometer_xout = read_word(0x3b)
accelerometer_yout = read_word(0x3d)
accelerometer_zout = read_word(0x3f)
f_output.write(struct.pack(bin_format, timestamp, accelerometer_xout, accelerometer_yout, accelerometer_zout))
Then later on, you could then convert the binary file to a CSV file using:
from datetime import datetime
import csv
import struct
bin_format = 'L3h' # Read data as signed words
entry_size = struct.calcsize(bin_format)
with open('accel-data.bin', 'rb') as f_input, open('accel-data.csv', 'wb') as f_output:
csv_output = csv.writer(f_output)
csv_output.writerow(['Time', 'X_Axis', 'Y_Axis', 'Z_Axis'])
while True:
bin_entry = f_input.read(entry_size)
if len(bin_entry) < entry_size:
break
entry = list(struct.unpack(bin_format, bin_entry))
entry[0] = datetime.fromtimestamp(entry[0]).strftime('%Y-%m-%d %H:%M:%S')
csv_output.writerow(entry)
If your data collection is not continuous, you could make use of threads. One thread would read your data into a special queue. Another thread could read items out of the queue onto the disk.
If it is continuous this approach will fail if the writing of data is slower than the reading of it.
Take a look at the special Format characters used to tell struct how to pack and unpack the binary data.

Windows pipes: Write from C - read in Python

I'd like to transmit a few bytes of data though a pipe to plot it from python.
I started with some snippets I found here but I cant get them working.
I've created the pipe like this:
int main(void){
HANDLE hPipe;
char buffer[24];
DWORD dwRead;
hPipe = CreateNamedPipe(TEXT("\\\\.\\pipe\\Pipe"),
PIPE_ACCESS_DUPLEX | PIPE_TYPE_BYTE | PIPE_READMODE_BYTE, // FILE_FLAG_FIRST_PIPE_INSTANCE is not needed but forces CreateNamedPipe(..) to fail if the pipe already exists...
PIPE_WAIT,
1,
24 * 16,
24 * 16,
NMPWAIT_USE_DEFAULT_WAIT,
NULL);
while (hPipe != INVALID_HANDLE_VALUE)
{
if (ConnectNamedPipe(hPipe, NULL) != FALSE) // wait for someone to connect to the pipe
{
while (ReadFile(hPipe, buffer, sizeof(buffer) - 1, &dwRead, NULL) != FALSE)
{
/* add terminating zero */
buffer[dwRead] = '\0';
/* do something with data in buffer */
printf("%s", buffer);
}
}
DisconnectNamedPipe(hPipe);
}
return 0;}
If I execute the following code it writes but the read part blocks:
import time
import struct
f = open(r'\\.\\pipe\\Pipe', 'r+b', 0)
i = 1
sss='ccccc'
while True:
s = sss.format(i)
i += 1
f.write(struct.pack('I', len(s)) + s) # Write str length and str
f.seek(0) # EDIT: This is also necessary
print 'Wrote:', s
n = struct.unpack('I', f.read(4))[0] # Read str length
s = f.read(n) # Read str
f.seek(0) # Important!!!
print 'Read:', s
time.sleep(2)
I tried commenting the ReadFile part in the C code but It did not work. Is there any other way to achieve this? I want to write from C and read from python. I tried writing into the pipe with CreateFile (from C) and it worked as expected. I only need the read part with python.

On most systems pipe is one-directional and you use two pipes to get two-directional (bidirectional) connection.
In your Python code you can open two connections
and then you don't need seek
import time
import struct
wf = open(r'Pipe', 'wb', 0)
rf = open(r'Pipe', 'rb', 0)
i = 0
template = 'Hello World {}'
while True:
i += 1
text = template.format(i)
# write text length and text
wf.write(struct.pack('I', len(text)))
wf.write(text)
print 'Wrote:', text
# read text length and text
n = struct.unpack('I', rf.read(4))[0]
read = rf.read(n)
print 'Read:', read
time.sleep(2)
EDIT: tested on Linux Mint 17, Python 3.4 & 2.7

I've solved it with PyWin32(http://sourceforge.net/projects/pywin32/files/) which seems to be the right tool for windows. I would rather use something more cross-plataform oriented but it has solved the problem.

PySerial and Value Splitting Error

I have encountered an error that has been seriously bothering me, and I have been unable to find a workaround. I have a python program that utilizes PySerial to import serial port values from a pulse oximeter. The problem is, when the function ser.readline() is invoked (in other words, when the python program is told to read values from the Arduino's serial monitor, the Arduino's serial values become distorted and the program returns a value unpack error.
Here is the python program.
import serial
import time
import pylab
import matplotlib.pyplot as plt
import numpy as np
import os
import csv
#time load
timestr = time.strftime("%Y_%m_%d")
#establish serial connection with ACM0
ser = serial.Serial('/dev/ttyACM0', 115200)
#establish variables
thymeL = [ ]
bpmL = [ ]
sp02L = [ ]
array_data = thymeL, bpmL, sp02L
#declare time
thyme = 1
#graph attributes
plt.ion()
plt.title("Pulse [BPM] & SPo2 [%] v. Time [s]", fontsize = "16")
plt.xlabel("Time [s]", fontsize = "14")
plt.ylabel("Pulse (red) [BPM] & SPo2 (blue) [%]", fontsize = "14")
while True:
data_in = ser.readline()
print data_in
data_in = data_in.strip('\n')
bpm,sp02 = data_in.split(",")
#convert string vals to float
thyme = float(thyme)
bpm = float(bpm)
sp02 = float(sp02)
#print to terminal
print "Time [s]: %s" % (thyme)
print "HR [BPM]: %s" % (bpm)
print "SPO2 [%%]: %s" % (sp02)
print
#append vectors
thymeL.append(thyme)
bpmL.append(bpm)
sp02L.append(sp02)
#print values to plot
plt.scatter(thyme,bpm,color="red")
plt.scatter(thyme,sp02,color="blue")
plt.pause(0.1)
time.sleep(0.05)
#update time
thyme = thyme + 0.5
#write to .csv
with open(full_path, 'w') as f:
writer = csv.writer(f)
for t, b, s in zip(array_data[0], array_data[1], array_data[2]):
writer.writerow([t, b, s])
The most important snippet of all is:
while True:
data_in = ser.readline()
print data_in
data_in = data_in.strip('\n')
bpm,sp02 = data_in.split(",")
The Arduino program is as follows:
#include <PinChangeInt.h>
#include <eHealth.h>
int cont = 0;
void setup() {
Serial.begin(115200);
eHealth.initPulsioximeter();
PCintPort::attachInterrupt(6, readPulsioximeter, RISING);
}
void loop() {
char buffer[32]; // make sure buffer is large enough
sprintf(buffer,"%d,%d \n",eHealth.getBPM(),eHealth.getOxygenSaturation());
Serial.print(buffer);
delay(500);
}
//=========================================================================
void readPulsioximeter(){
cont ++;
if (cont == 50) { //Get only of one 50 measures to reduce the latency
eHealth.readPulsioximeter();
cont = 0;
}
}
So, the serial monitor is outputting values like this:
67,95
66,95
67,96
and so on.
But only when ser.readline() is invoked, the values become skewed and are unable to be unpacked by the split(',') function. In the following photo (1) and (2), you can see the distortion of the values right when the ser.readline() is called.
How can I re-word the python OR Arduino program in such a way to circumvent this distortion and allow the values to be split and unpacked without any errors?

So is there any possibility that something else is calling loop() asynchronously, e.g. from an interrupt routine, which might try to simultaneously transmit another string of readings at the same time as the arduino's 'mainline' calling of the loop function? An aside: if the interrupt routine readPulsioximeter() got invoked while the mainline loop() function was between the calls to eHealth.getBPM() and eHealth.getOxygenSaturation() and updated the Pulsioximeter attributes, does your code guarantee to send values from the same reading out onto the serial port?