Develop Reference

High data-rate parallelism between python and arduino uno

I'm working on a project where a data capture system sends EMG data to a python script through an Arduino Uno's serial port and the python script classifies the user's intent by processing a chunk of that data. The data capture system sends samples at 250Hz and I require my python script to not miss samples the capture system is sending while classifying.
For this I want two parallelly running processes:
A process continuously capturing the data and buffering it. (using PySerial) - running every 0.004s
A process taking data from the buffer, preprocess it and classifying it. - this process approximately takes 0.03 seconds to run per loop excluding data acquisition
I don't know how to proceed with this. I've tried and read through a lot of articles but to no avail.
I tried threading, but apparently threading is not true parallelism, so I gave up trying to make it work.
I tried using multiprocessing but the multiprocessing module doesn't like pickling pyserial.
Here's what I tried: (It's probably full of mistakes, I hacked it up but I expected some output, also this is not indefinitely buffering, but I'm trying to get at least the first buffer printed)
Driver Code:
import GatherData as gd
import serial.tools.list_ports
from multiprocessing import Pool
if __name__ == '__main__':
arr = []
serialInst = serial.Serial()
ports = serial.tools.list_ports.comports()
portList = []
for Port in ports:
portList.append(str(Port))
print(str(Port))
val = input("Select Port: COM")
for x in range(0,len(portList)):
if(portList[x].startswith("COM"+str(val))):
portVar = "COM"+str(val)
print(portList[x])
serialInst.baudrate = 115200
serialInst.port = portVar
serialInst.open()
p = Pool(6) # Number of concurrent processes
arr = p.starmap(gd.collect, (arr, serialInst)) # Start all processes
p.map(gd.printnum(arr))
p.close()
p.join()
GatherData.py:
import serial.tools.list_ports
import numpy as np
import pandas as pd
# import Dependencies as dp
def collect(array,serialInst):
ch = 4
if not array:
return makearr()
else:
nextw = array[1:]
temparr = []
if (serialInst.in_waiting):
packet = serialInst.readline()
# now = time.time()
line_as_list = packet.split(b'\t')
#print(line_as_list)
for i in range(1,ch+1):
try:
rand1=line_as_list[i]
#print("rand1", rand1)
rand1List = rand1.split(b'\n')
rand1f = float(rand1List[0])
temparr.append(rand1f)
#print(temparr)
except IndexError:
return collect(array) #add pca
except ValueError:
return collect(array) #add pca
nextw.append(temparr)
print(nextw)
return nextw
def makearr(serialInst):
final = []
counter = 0
while(counter<50):
temparr = []
if (serialInst.in_waiting):
packet = serialInst.readline()
line_as_list = packet.split(b'\t')
flag = 0
for i in range(1,5):
try:
rand1=line_as_list[i]
#print("rand1", rand1)
rand1List = rand1.split(b'\n')
rand1f = float(rand1List[0])
temparr.append(rand1f)
except IndexError:
flag = 1
continue
except ValueError:
flag = 1
continue
if(flag == 0):
final.append(temparr)
counter=counter+1
return final
def printnum(array):
while True:
if array:
print(len(array), end='\r')
The errors:
COM8 - Arduino Uno (COM8)
Select Port: COM8
COM8 - Arduino Uno (COM8)
Traceback (most recent call last):
File "D:\anaconda\envs\tf\lib\site-packages\spyder_kernels\py3compat.py", line 356, in compat_exec
exec(code, globals, locals)
File "c:\users\indra\documents\igibup.py", line 34, in <module>
arr = p.starmap(gd.collect, (arr, serialInst)) # Start all processes
File "D:\anaconda\envs\tf\lib\multiprocessing\pool.py", line 276, in starmap
return self._map_async(func, iterable, starmapstar, chunksize).get()
File "D:\anaconda\envs\tf\lib\multiprocessing\pool.py", line 657, in get
raise self._value
File "D:\anaconda\envs\tf\lib\multiprocessing\pool.py", line 431, in _handle_tasks
put(task)
File "D:\anaconda\envs\tf\lib\multiprocessing\connection.py", line 206, in send
self._send_bytes(_ForkingPickler.dumps(obj))
File "D:\anaconda\envs\tf\lib\multiprocessing\reduction.py", line 51, in dumps
cls(buf, protocol).dump(obj)
ValueError: ctypes objects containing pointers cannot be pickled
I don't understand how to proceed.

minimalmodbus checksum error in rtu mode with 2 devices

I have a RPI with a USB CH340 dongle connected to a EM340 energy meter. It works fine with the code below.
When I connect 2 x EM340 energy meters I get the following error:
pi#raspberrypi:~ $ python3 modbus_test.py
Traceback (most recent call last):
File "modbus_test.py", line 21, in <module>
freq2 = instrument.read_register(0x0033,1) # Registernumber, number of decimals
File "/home/pi/.local/lib/python3.7/site-packages/minimalmodbus.py", line 486, in read_register
payloadformat=_Payloadformat.REGISTER,
File "/home/pi/.local/lib/python3.7/site-packages/minimalmodbus.py", line 1245, in _generic_command
payload_from_slave = self._perform_command(functioncode, payload_to_slave)
File "/home/pi/.local/lib/python3.7/site-packages/minimalmodbus.py", line 1330, in _perform_command
response, self.address, self.mode, functioncode
File "/home/pi/.local/lib/python3.7/site-packages/minimalmodbus.py", line 1867, in _extract_payload
raise InvalidResponseError(text)
minimalmodbus.InvalidResponseError: Checksum error in rtu mode: '6ý' instead of '\x99ò' . The response is: '\x01\x01\x00\x00\x166ý' (plain response: '\x01\x01\x00\x00\x166ý')
My code:
import minimalmodbus
import serial
instrument = minimalmodbus.Instrument('/dev/ttyUSB0', 1) # port name, slave address (in decimal)
#instrument.serial.port # this is the serial port name
instrument.serial.baudrate = 9600 # Baud
instrument.serial.bytesize = 8
#instrument.serial.parity = serial.PARITY_NONE
instrument.serial.stopbits = 1
instrument.serial.timeout = 0.2 # seconds
instrument.address = 1 # this is the slave address number
#instrument.mode = minimalmodbus.MODE_ASCII # rtu or ascii mode
instrument.mode = minimalmodbus.MODE_RTU
instrument.clear_buffers_before_each_transaction = True
instrument.close_port_after_each_call = True
## Read temperature (PV = ProcessValue) ##
freq2 = instrument.read_register(0x0033,1) # Registernumber, number of decimals
txt = "Frekvens: {} Hz".format(freq2)
#0x0034
kwh_total = instrument.read_register(0x0400)
kwh_total_2 = instrument.read_register(0x0402)
txt2 = "Total forbrug: {0}.{1} kWh".format(kwh_total,kwh_total_2)
w_l1 = instrument.read_register(0x0012,1) # Registernumber, number of decimals
txt3 = "Nuværende forbrug: {} W".format(w_l1)
print(txt)
print(txt2)
print(txt3)
I have wired the 2xEM340 according to the https://web.evishine.dk/wp-content/uploads/2019/11/EM340-ENG.pdf page 3.
Any idea why I get Checksum error in rtu mode ?

As per the comments if you attach two Modbus RTU devices with the same ID in parallel then they will both respond to any request addressed to that Slave ID. The responses will probably collide which means your code will receive a garbled response (detected via the CRC).
The solution is to change the ID of one of the devices.

Read the serial port

I dont seem to get the proper reading Id like from my serial.write/read. What do I do wrong?
After one turn in the loop, I would like to have another A00 output, but instead I get D13HIGH, as you can see below.
import serial
from time import sleep
import math
port = '/dev/ttyAMA0'
baud = 9600
ser = serial.Serial(port=port, baudrate=baud)
sleep(0.2)
count = 0
while count < 20:
ser.write('a--A00READ--')
sleep(0.2)
reply = ser.read(12)
print(reply)
adc = reply[7:]
adc = adc.strip('-')
adc = int(adc)
volts = (adc / 1023.0 * 5.0)
ser.write('a--D13HIGH--')
count += 1
ser.close()
Output:
>>>
a--A00+487--
a--D13HIGH--
Traceback (most recent call last):
File "/home/pi/serialcom.py", line 21, in <module>
adc = int(adc)
ValueError: invalid literal for int() with base 10: 'IGH'
>>>

passing strings to functions in python

I am trying to use a variable to pass a string to the serial.Serial function in python. The following works the way I want it to:
port = serial.Serial("/dev/ttyAMA0", baudrate=9600,bytesize=serial.SEVENBITS,stopbits=serial.STOPBITS_ONE,parity=serial.PARITY_EVEN,timeout=3.0)
However if I try to put the string into a variable like so and then call the function with the variable I get an error.
serialString = '"/dev/ttyAMA0",baudrate=9600,bytesize=serial.SEVENBITS,stopbits=serial.STOPBITS_ONE,parity=serial.PARITY_EVEN,timeout=3.0'
port = serial.Serial(serialString)
Here is the error:
Traceback (most recent call last): File "./cncserver.py", line 34,
in
port = serial.Serial(serialString) File "/usr/lib/python2.7/dist-packages/serial/serialutil.py", line 260, in
init
self.open() File "/usr/lib/python2.7/dist-packages/serial/serialposix.py", line 276, in
open
raise SerialException("could not open port %s: %s" % (self._port, msg)) serial.serialutil.SerialException: could not open port
"/dev/ttyAMA0",baudrate=9600,bytesize=serial.SEVENBITS,stopbits=serial.STOPBITS_ONE,parity=serial.PARITY_EVEN,timeout=3.0:
[Errno 2] No such file or directory:
'"/dev/ttyAMA0",baudrate=9600,bytesize=serial.SEVENBITS,stopbits=serial.STOPBITS_ONE,parity=serial.PARITY_EVEN,timeout=3.0'
I'm guessing it's something to do with the filepath in the string not being quoted properly but I've spent enough time trying to figure it out, can somebody help me here please?
Thanks,
Lorne
Here is my revised code:
#parameter settings of the serial port on the PI must match the control
serialPort = "/dev/ttyAMA0"
serialBaudRate = 9600
serialByteSize = serial.SEVENBITS
serialStopBits = serial.STOPBITS_ONE
serialParity = serial.PARITY_EVEN
serialTimeout = 3.0
port = serial.Serial(serialPort,serialBaudRate,serialByteSize,serialStopBits,serialParity,serialTimeout)
Here is the new error:
Traceback (most recent call last): File "./cncserver.py", line 39,
in
port = serial.Serial(serialPort,serialBaudRate,serialByteSize,serialStopBits,serialParity,serialTimeout)
File "/usr/lib/python2.7/dist-packages/serial/serialutil.py", line
250, in init
self.parity = parity File "/usr/lib/python2.7/dist-packages/serial/serialutil.py", line 344, in
setParity
if parity not in self.PARITIES: raise ValueError("Not a valid parity: %r" % (parity,)) ValueError: Not a valid parity: 1

port = serial.Serial("/dev/ttyAMA0", baudrate=9600,bytesize=serial.SEVENBITS,stopbits=serial.STOPBITS_ONE,parity=serial.PARITY_EVEN,timeout=3.0)
is not the same as
port = serial.Serial('"/dev/ttyAMA0",baudrate=9600,bytesize=serial.SEVENBITS,stopbits=serial.STOPBITS_ONE,parity=serial.PARITY_EVEN,timeout=3.0')
This is not a string, but function parameters that have been passed as key_pair values.
"/dev/ttyAMA0", baudrate=9600,bytesize=serial.SEVENBITS,stopbits=serial.STOPBITS_ONE,parity=serial.PARITY_EVEN,timeout=3.0
sends 6 different parameters to the function, :-
/dev/ttyAMA0
baudrate as 9600
bytesize as serial.SEVENBITS
stopbits as serial.STOPBITS_ONE
parity as serial.PARITY_EVEN
timeout as 3.0
'"/dev/ttyAMA0",baudrate=9600,bytesize=serial.SEVENBITS,stopbits=serial.STOPBITS_ONE,parity=serial.PARITY_EVEN,timeout=3.0'
sends only one parameter as
'"/dev/ttyAMA0",baudrate=9600,bytesize=serial.SEVENBITS,stopbits=serial.STOPBITS_ONE,parity=serial.PARITY_EVEN,timeout=3.0'
as for what you are trying to do,
initialize :
args = ("/dev/ttyAMA0",)
kwargs= {'baudrate':9600,'bytesize':serial.SEVENBITS,'stopbits':serial.STOPBITS_ONE,'parity':serial.PARITY_EVEN,'timeout':3.0}
and call them wherever you want
port = serial.Serial(*args, **kwargs)

python error: [Errno 32] Broken pipe in Socket.sendall()

I am writing a multi-threaded client/server program. It splits a large file into smaller files in its client side and sends the smaller files to the server concurrently.
The problem is that in every run, the server can only receive two of the smaller files (the first one and another random one). Meanwhile, I encounter the error: "[Errno 32] Broken pipe" in client side of the program in s.sendall(part). The error arises in every thread that starts to send one of the smaller files before reception of the first file (on the server). In other words, every thread that starts to send after the reception the first file (on the server) can complete its task.
I run each of the client and server codes on different computers (both have the following specification: Ubuntu 14.04 desktop, 64 bit, 16GiB ram)
Client side Error:
Traceback (most recent call last):
File "Desktop/File_transmission/Client.py", line 56, in sendSplittedFile
s.sendall(part)
File "/usr/lib/python2.7/socket.py", line 224, in meth
return getattr(self._sock,name)(*args)
error: [Errno 32] Broken pipe
Client.py:
import random
import socket
import time
import threading
import errno
import select
import File_manipulation
import sys, traceback
class Client:
nodesIpAddr = ["....", "...."] #Server = ....
dataPort = 45678
delay = 2
inputFileAddress = 'tosend.dat'
fileOutputPrefix = 'output'
fileOutputSuffix = ".dat"
bufferSize = 2048
max_size_splitted_file = 10*(2**20) # 10 MiB
def __init__ (self, my_ip):
self.ip = my_ip
def send(self, ip_toSend, dataPort):
print "\tSend function is runing."
totalNumSplittedFile = File_manipulation.split_file(Client.inputFileAddress, Client.fileOutputPrefix, Client.max_size_splitted_file , Client.bufferSize)
for i in range(0, totalNumSplittedFile):
thread_send = threading.Thread(target = self.sendSplittedFile, args = (ip_toSend, dataPort, Client.bufferSize, i, Client.fileOutputPrefix, totalNumSplittedFile))
thread_send.start()
def sendSplittedFile(self, ip_toSend, dataPort, bufferSize, fileNumber, fileNamePrefix, totalNumSplittedFile):
# Create a socket (SOCK_STREAM means a TCP socket)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
BUFFER_SIZE = bufferSize
try:
s.connect((ip_toSend, dataPort))
f = open(fileNamePrefix + '.%s' % fileNumber,'rb')
s.send(str(fileNumber) + " " + str(totalNumSplittedFile))
part = f.read(BUFFER_SIZE)
while (part):
s.sendall(part)
part = f.read(BUFFER_SIZE)
f.close()
s.sendall(part)
time.sleep(Client.delay)
s.sendall('EOF')
print "Done Sending."
print s.recv(BUFFER_SIZE)
s.close()
print "\tData is sent to ", ip_toSend,
except socket.error, v:
traceback.print_exception(*sys.exc_info())
s.close()
nodeIP = [(s.connect(('8.8.8.8', 80)), s.getsockname()[0], s.close()) for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]][0][1]
n = Client(nodeIP)
n.send(n.nodesIpAddr[0], n.dataPort)
Server Side Error:
Traceback (most recent call last):
File "/usr/lib/python2.7/SocketServer.py", line 295, in _handle_request_noblock
self.process_request(request, client_address)
File "/usr/lib/python2.7/SocketServer.py", line 321, in process_request
self.finish_request(request, client_address)
File "/usr/lib/python2.7/SocketServer.py", line 334, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/usr/lib/python2.7/SocketServer.py", line 649, in __init__
self.handle()
File "Desktop/File_transmissionServer.py", line 37, in handle
totalFileNumber = int(details[1])
ValueError: null byte in argument for int()
Server.py
import socket
import time
import threading
import errno
import select
import SocketServer
import File_manipulation
class ServerThreadHandler(SocketServer.BaseRequestHandler):
nodesIpAddr = ["....", "...."] #Server = ....
fileOutputPrefix = 'torec '
fileOutputSuffix = '.dat'
dataPort = 45678
delay = 3
maxNumClientListenedTo = 200
timeout_in_seconds = 5
bufferSize = 2048
totalFileNumber = 0 #Total number of splitted files. It should be set by the incoming packets
def handle(self):
BUFFER_SIZE = ServerThreadHandler.bufferSize # Normally 1024, but we want fast response
# self.request is the TCP socket connected to the client
data = self.request.recv(BUFFER_SIZE)
addr = self.client_address[0]
details = str(data).split()
currentFileNum = int(details[0])
totalFileNumber = int(details[1])
print '\tReceive: Connection address:', addr,'Current File Number: ', currentFileNum, 'Total Number of splitted files: ', totalFileNumber
f = open(ServerThreadHandler.fileOutputPrefix + '_Received.%s' % currentFileNum, 'wb')
data = self.request.recv(BUFFER_SIZE)
while (data and data != 'EOF'):
f.write(data)
data = self.request.recv(BUFFER_SIZE)
f.close()
print "Done Receiving." ," File Number: ", currentFileNum
self.request.sendall('\tThank you for data. File Number: ' + str(currentFileNum))
if __name__ == "__main__":
HOST, PORT = ServerThreadHandler.nodesIpAddr[0], ServerThreadHandler.dataPort # HOST = "localhost"
server = SocketServer.TCPServer((HOST, PORT), ServerThreadHandler)
# Activate the server; this will keep running until you interrupt the program with Ctrl-C
server.serve_forever()