Develop Reference

Python - Subprocess - Recording and Dumping Measurements

In python3 with Ubuntu 16.04LTS, I have a subprocess that I created from my main script to record measurements from a device connected to my local machine. I would like to know how to send a message to this subprocess when I want to finish data recording, and switch to dumping the measurements to a csv file. Shown below is a stripped-down version of what I have tried so far, but the code hangs and I am unable to dump the measurements I record. In fact, I only record 1 measurement. I am not sure about how to asynchronously check for stdin inputs while recording data. May I please get some help?
Main.py
# start subprocess
p_1 = subprocess.Popen(["./ekg.py", saveFilename_ekg], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# do other stuff
...
# send message to quit
message = str("1")
encMsg = message.encode()
print("Message:", encMsg.decode())
p_stdout = p_1.communicate(input=encMsg)[0]
# print "Done" from subprocess
print(p_stdout.decode('utf-8').strip())
# kill subprocess
p_1.kill()
ekg.py
def dumpLiveData(outputFile):
ekg = ekgClass()
dataMeasurements = []
for liveData in ekg.getLiveData():
# monitor stdin for message
if int(sys.stdin.read()) == 1:
break
else:
meas = [liveData.time, liveData.pulseWaveform]
dataMeasurements.append(meas)
#print ("Dumping data")
with open(outputFile, 'wb') as csvfile:
writer = csv.writer(csvfile, quoting=csv.QUOTE_NONNUMERIC)
#print ("Created text file")
header = ["Time", "Waveform value"]
writer.writerow(header)
for idx, val in enumerate(dataMeasurements):
writer.writerow(dataMeasurements[idx])
print("Done")
if __name__== "__main__":
# get parameters
parser = argparse.ArgumentParser(description="ekg.py")
parser.add_argument("outputFile", help="Output CSV file.")
# parse
args = parser.parse_args()
# record and dump measurements
dumpLiveData(args.outputFile)

Solved it by sending a control + C event to the subprocess. An try-except-else block caught the keyboard interrupt, processed it, and then I gracefully exit the block. After exiting, I write the data recorded to a csv file.
main.py
import subprocess, signal
# start subprocess
p_1 = subprocess.Popen(["./ekg.py", saveFilename_ekg], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# do other stuff
...
# send control + C event
p_1.send_signal(signal.SIGINT)
stdout, stderr = p_1.communicate(input=encMsg)[0]
# print output from subprocess
print(stdout.decode('utf-8').strip())
# wait for subprocess to write file
p_1.wait()
# kill subprocess
p_1.kill()
ekg.py
def dumpLiveData(outputFile):
ekg = ekgClass()
dataMeasurements = []
exception_found = None
try:
for liveData in ekg.getLiveData():
if exception_found == True:
break
meas = [liveData.time, liveData.pulseWaveform]
dataMeasurements.append(meas)
except KeyboardInterrupt:
exception_found = True
else:
pass
print ("Dumping data")
with open(outputFile, 'wb') as csvfile:
writer = csv.writer(csvfile, quoting=csv.QUOTE_NONNUMERIC)
print ("Created text file")
header = ["Time", "Waveform value"]
writer.writerow(header)
for idx, val in enumerate(dataMeasurements):
writer.writerow(dataMeasurements[idx])
print("Done")

Python: "while" is too slow, and sleep(0.25) becomes sleep(3) in actual execution

I am running a Python Program on a Raspberry Pi 3 which I want to log the temperature from a DS18B20 sensor once every 0.25 seconds.
Earlier, when the program was simple and displaying the temperature on shell, it was quite fast and not having issues. Unfortunately due to the program itself now which includes logging to a file, I am getting a log every 2 seconds or 3 seconds only.
How do I ensure the 0.25 second logging interval.
I have shared the code below:
#This program logs temperature from DS18B20 and records it
#Plots the temperature-time plot.
import os
import sys
#import matplotlib.pyplot as plt
from re import findall
from time import sleep, strftime, time
from datetime import *
#plt.ion()
#x = []
#y = []
ds18b20 = ''
def setup():
global ds18b20
for i in os.listdir('/sys/bus/w1/devices'):
if i != 'w1_bus_master1':
ds18b20 = i
# Reads temperature data from the Temp sensor
# This needs to be modified for use with max31855 and K-type thermocouples
def read():
# global ds18b20
location = '/sys/bus/w1/devices/' + ds18b20 + '/w1_slave'
tfile = open(location)
text = tfile.read()
tfile.close()
secondline = text.split("\n")[1]
temperaturedata = secondline.split(" ")[9]
temperature = float(temperaturedata[2:])
temperature = temperature / 1000
return temperature
#Loop for logging - sleep, and interrupt to be configured.
def loop():
while True:
if read() != None:
print "Current temperature : %0.3f C" % read()
#sleep(0.25)
func()
def write_temp(temperature,file_name):
with open(file_name, 'a') as log:
log.write("{0},{1}\n".format(datetime.now().strftime("%d-%m-%Y %H:%M:%S"),str(temperature)))
arg = sys.argv[1]
filename1 = str(arg) + "-" + datetime.now().strftime("%d-%m-%Y-%H-%M-%S")+".csv"
def func():
temperature = read()
#sleep(0.25)
write_temp(temperature,filename1)
#graph(temperature)
#For plotting graph using MatPlotLib
#Comment out this function during foundry trials to avoid system slowdown
#Check system resource usage and slowdown using TOP or HTOP
#def graph(temperature):
# y.append(temperature)
# x.append(time())
# plt.clf()
# plt.scatter(x,y)
# plt.plot(x,y)
# plt.draw()
#Interrupt from command-line
def destroy():
pass
if __name__ == '__main__':
try:
setup()
func()
loop()
except KeyboardInterrupt:
destroy()
I have commented out sections that I thought to be resource heavy, but still I can't manage anything less than 2 seconds. I am getting results as below:
Output:
27-09-2016 12:18:41,23.0
27-09-2016 12:18:43,23.062
27-09-2016 12:18:46,23.125
27-09-2016 12:18:48,23.187
27-09-2016 12:18:50,23.187
27-09-2016 12:18:53,23.562
27-09-2016 12:18:55,25.875
27-09-2016 12:18:58,27.187
27-09-2016 12:19:00,27.5

Only open the logfile once (and close it on program exit)
Don't always re-read the temperature from the sensor. You call read() way too often.
Reduce general overhead and simplify your calls.
I am not able to completely test this, but something like this sould work:
import os
import sys
import time
from datetime import datetime
def read_temp(dev):
'''Reads temperature from sensor and returns it as float.'''
loc = '/sys/bus/w1/devices/' + dev + '/w1_slave'
with open(loc) as tf:
return float(tf.read().split('\n')[1].split(' ')[9][2:]) / 1000.0
def write_temp(t, logfile):
'''Writes temperature as .3 float to open file handle.'''
logfile.write('{0},{1:.3f}\n'.format(datetime.now().strftime('%d-%m-%Y %H:%M:%S'), t))
def loop(dev, logfile):
'''Starts temperature logging until user interrupts.'''
while True:
t = read_temp(dev)
if t:
write_temp(t, logfile)
print('Current temperature: {0:.3f} °C'.format(t))
sys.stdout.flush() # Flush. Btw, print is time-consuming!
time.sleep(.25)
if __name__ == '__main__':
# Take the first match for a device that is not 'w1_bus_master1'
dev = [d for d in os.listdir('/sys/bus/w1/devices') if d != 'w1_bus_master1'][0]
# Prepare the log filename
fname = str(sys.argv[1]) + "-" + datetime.now().strftime("%d-%m-%Y-%H-%M-%S")+".csv"
# Immediately open the log in append mode and do not close it!
logfile = open(fname, 'a')
try:
# Only pass device and file handle, not the file name.
loop(dev, logfile)
except KeyboardInterrupt:
# Close log file on exit
logfile.close()

Calling a thread out of scope in python

I have a thread that is defined as in a program that continuously reads serial data along with running a UI in wxpython.
dat = Thread(target=receiving, args=(self.ser,))
The method it calls "receiving" runs in an infinite loop
def receiving(ser):
global last_received
buffer = ''
while True:
date = datetime.date.today().strftime('%d%m%Y')
filename1 = str(date) + ".csv"
while date == datetime.date.today().strftime('%d%m%Y'):
buffer = buffer + ser.read(ser.inWaiting())
if '\n' in buffer:
lines = buffer.split('\n')
if lines[-2]:
last_received = lines[-2]
buffer = lines[-1]
print_data =[time.strftime( "%H:%M:%S"), last_received]
try:
with open(filename1, 'a') as fob:
writ = csv.writer(fob, delimiter = ',')
writ.writerow(print_data)
fob.flush()
except ValueError:
with open('errors.log','a') as log:
log.write('CSV file writing failed ' + time.strftime("%H:%M:%S")+' on '+datetime.date.today().strftime('%d/%m/%Y')+'\n')
log.close()
The argument is defined as
class SerialData(object):
def __init__(self, init=50):
try:
serial_list = serialenum.enumerate()
self.ser = ser = serial.Serial(
port=serial_list[0],
baudrate=9600,
bytesize=serial.EIGHTBITS,
parity=serial.PARITY_NONE,
stopbits=serial.STOPBITS_ONE,
timeout=None,
xonxoff=0,
rtscts=0,
interCharTimeout=None
)
except serial.serialutil.SerialException:
# no serial connection
self.ser = None
else:
dat = Thread(target=receiving, args=(self.ser,))
if not dat.is_alive:
dat.start()
def next(self):
if not self.ser:
# return anything so we can test when Serial Device isn't connected
return 'NoC'
# return a float value or try a few times until we get one
for i in range(40):
raw_line = last_received
try:
return float(raw_line.strip())
time.sleep(0.1)
except ValueError:
# print 'Not Connected',raw_line
time.sleep(0.1)
return 0
Due to a bug in Ubuntu 14.04 the thread hangs after a while. I wanted to periodically check if the thread is alive and start it again if it is not. So I did something like
def on_timer(self):
self.text.SetLabel(str(mul_factor*self.datagen.next()))
if not dat.is_alive():
dat.start()
wx.CallLater(1, self.on_timer)
This runs every second to update the data in UI but also needs to check if the thread is not stopped. But this gives me an error saying "NameError: global name 'dat' is not defined". I also tried referring to the thread using the object name path. But didn't work either.
Can someone help me as to how I can start the thread out of scope?

It seems like you want to replace dat with self.dat. dat only exists in the scope of the __init__ method. I suggest reading up on Python scoping rules.

how two process communicate with each other

I'm very new with python.
I started implementing twp daemon processes that will send messages to each other.
right now i have just 2 daemons that are running.
I don't understand how to build something that they can communicate through..
I read that there are pipe, or queue ...
sill, could not understand how to build a pipe or a queue that the two ends will be the two processes..
import multiprocessing
import time
import sys
def daemon():
p = multiprocessing.current_process()
print 'Starting:', p.name, p.pid
sys.stdout.flush()
while (1):
time.sleep(1)
print 'Exiting :', p.name, p.pid
sys.stdout.flush()
def machine_func():
p = multiprocessing.current_process()
print 'Starting:', p.name, p.pid
sys.stdout.flush()
while (1):
time.sleep(1)
print 'Exiting :', p.name, p.pid
sys.stdout.flush()
cs = multiprocessing.Process(name='control_service', target=control_service_func)
cs.daemon = True
m = multiprocessing.Process(name='machine', target=machine_func)
m.daemon = True
cs.start()
m.start()

You can find very good examples here: Communication Between Processes

you can communicate with daemons via text files like this:
from multiprocessing import Process
from ast import literal_eval as literal
from random import random
import time
def clock(): # 24 hour clock formatted HH:MM:SS
return str(time.ctime())[11:19]
def sub_a(): # writes dictionary that tallys +/- every second
a = 0
while 1:
data = {'a': a}
opened = 0
while not opened:
try:
with open('a_test.txt', 'w+') as file:
file.write(str(data))
opened = 1
except:
print ('b_test.txt in use, try WRITE again...')
pass
a+=1
time.sleep(random()*2)
def sub_b(): # writes dictionary that tallys +/- every 2 seconds
b = 0
while 1:
data = {'b': b}
opened = 0
while not opened:
try:
with open('b_test.txt', 'w+') as file:
file.write(str(data))
opened = 1
except:
print ('b_test.txt in use, try WRITE again...')
pass
b += 1
time.sleep(random()*4)
# clear communication lines
with open('a_test.txt', 'w+') as file:
file.write('')
with open('b_test.txt', 'w+') as file:
file.write('')
# begin daemons
sa = Process(target=sub_a)
sa.daemon = True
sb = Process(target=sub_b)
sb.daemon = True
sa.start()
sb.start()
begin = time.time()
m = 0
while 1:
m += 1
time.sleep(1)
elapsed = int(time.time()-begin)
#fetch data from deamons
opened = 0
while not opened:
try:
with open('a_test.txt', 'r') as f:
a = literal(f.read())
opened = 1
except:
print ('a_test.txt in use, try READ again...')
pass
opened = 0
while not opened:
try:
with open('b_test.txt', 'r') as f:
b = literal(f.read())
opened = 1
except:
print ('READ b_test.txt in use, try READ again...')
pass
print(clock(), '========', elapsed, b['b'], a['a'])
in this manner you can make object (like a dict) into string, write() to file, then:
ast.literal_eval
to get it back out on the other side when you read()
while not opened try
method prevents race condition so daemons and main process have time needed to not clash while they open/process/close the file
with open as file
method ensures file is opened and closed efficiently
added bonus is you can open the text file in an editor to check its state in real time.

Suspected thread issue with pyinotify

I have been working with pyinotify and I am having issues with it where after multiple changes to a folder it simply stops receiving notifications. I have a feeling it is something to do with the the fact that two threads are running; namely the notifier thread and the wxpython thread.
The purpose of the app is to essentially load a picture to the screen when it detects an ip connection, monitor a folder for the file 'Checklist' and based on that file do some processing i.e move files around.
It works intermittently but being a python newbie Im not exactly sure what the issue might be as I have basically taken the threaded example and worked around it. Sometimes, it only gets one notification and stops receiving file change notifications.
Additionally, if I restart the linux box and try again, it works for a good number of file changes and then stops receiving notifications again which makes me think that perhaps its not releasing the watches properly?
Any help would be greatly appreciated as well as optimizations and improvements are very welcome. I'm sure I could learn a lot from the feedback. The code is below
import pyinotify
import os.path
import shutil
import errno
import subprocess
import logging
import wx
import time
import signal
import sys
#update CHECKLIST name
CHECKLIST = 'Checklist' #this must exist in the update archive
#static values
DIR_UPDATE = 'd'
FILE_UPDATE = 'f'
PING_IP = ' localhost' # change in production
#paths
WATCH_PATH = '/home/test'
LOG_PATH = '/home/test/update.log'
CONNECTED_IMG = 'update.jpg'
UPDATING_IMG = 'updating.png'
#msgs
UPDATEFOUND_MSG = ' Update Found '
UPDATEUNZIPPEDSTART_MSG = ' Update unzipping '
UPDATEUNZIPPED_MSG = ' Update unzipped '
UPDATEFILE_MSG = ' Update file '
UPDATEFILEMSG_CONT = ' moved into path '
REMOVEFILEMSG_CONT = ' removed from update folder '
UPDATECOMPLETE_CONT = ' Update complete'
ROADANGELRESTART_MSG = ' Update restarting app '
DIRCREATED_MSG = ' Directory created at path '
#errors
UPDATEFAILED_MSG = ' Update process failed on '
BADLYFORMED_MSG = ' Badly formed src/dest combination '
UPDATESRCUNAVAILABLE = ' Invalid update file specified '
UPDATEDESTUNAVAILABLE = ' Invalid update destination specified '
INVALIDUPDATEFORMAT = ' Invalid format string '
#on startup create the watchfolder if it doesnt exist
WM = pyinotify.WatchManager() # Watch Manager
WM_MASK = pyinotify.IN_CLOSE_WRITE # watched events
#setup logger
LOGGER = logging.getLogger('Updater')
LOG_HANDLE = logging.FileHandler(LOG_PATH)
FORMATTER = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
LOG_HANDLE.setFormatter(FORMATTER)
LOGGER.addHandler(LOG_HANDLE)
LOGGER.setLevel(logging.INFO)
#Global values used primarily in the main function loop
HANDLER = None
NOTIFIER = None
WDD = None
UPDATE_UI = None
WINDOW = None
CURR_IMG = None
LAST_CURRIMG = None
class EventHandler(pyinotify.ProcessEvent):
VERBOSE = False
""" Main class to monitor file events and process accordingly"""
def process_IN_CLOSE_WRITE(self, event):
""" Only executes when a Checklist has finished being written to"""
path = event.pathname
print 'evt'
#look for the update_ready file before processing
if (os.path.basename(path) == 'Checklist'):
EventHandler.parse_updates(WATCH_PATH)
global CURR_IMG
CURR_IMG = os.path.join(WATCH_PATH, UPDATING_IMG)
show_window()
print 'update completed'
time.sleep(1000)
#classmethod
def parse_updates(cls, path):
""" parses update files """
#handle errors for opening the file
file_path = os.path.join(path, CHECKLIST)
print file_path
files = open(file_path)
#handle errors for malformed tuples-done
#handle errors for unavailable files-done
#handle errors for unavailable dests-done #created automatically
#handle permission errors
for line in files:
#remove linebreaks etc and ensure its not empty
if line.strip():
array = line.split('=')
length = len(array)
if length == 3:
EventHandler.process_line(path, array)
else:
if length > 0:
EventHandler.print_bad_msg(array[0])
else:
EventHandler.print_bad_msg()
print 'removing ', file_path
os.remove(file_path) #remove the checklist file
#classmethod
def mkdir(cls, path):
""" makes a directory from a path"""
try:
os.mkdir(path)
print DIRCREATED_MSG, path
except OSError, err:
print err
if err.errno != errno.EEXIST: #thrown when the dir already exists
return False
#classmethod
def move_file(cls, src, dest):
""" moves a file from src to dest and remove after
expects that the dest already exists at this point
otherwise ignores the move"""
#print 'moving from', src, 'to ', dest
if os.path.isfile(dest):
shutil.copy2(src, dest)
else:
print UPDATEDESTUNAVAILABLE
#remove the src file when done
os.remove(src)
#classmethod
def process_line(cls, path, array):
""" process a line from the checklist"""
#remove newlines etc
update_file = array[0].strip()
update_src = os.path.join(path, update_file)
update_dest = array[1].strip()
update_type = array[2].strip()
#ensure we have valid values in all three fields
if update_file and update_dest and update_type:
#ensure the src file exists
if os.path.isfile(update_src):
#check if destination is directory and
#copy the file into the directory
if update_type == DIR_UPDATE:
EventHandler.mkdir(update_dest)
dest = os.path.join(update_dest, update_file)
EventHandler.move_file(update_src, dest)
else:
EventHandler.move_file(update_src, update_dest)
else:
print UPDATESRCUNAVAILABLE
else:
print INVALIDUPDATEFORMAT
#classmethod
def print_bad_msg(cls, msg = ''):
""" print a badly formed message with optional value"""
if msg:
print BADLYFORMED_MSG, msg
else:
print BADLYFORMED_MSG
class UpdateFrame(wx.Frame):
""" Displays update images to screen"""
def __init__(self, path):
wx.Frame.__init__(self, None, wx.ID_ANY)
image_file = path
image = wx.Bitmap(image_file)
image_size = image.GetSize()
# set the frame size to fit the screen size
self.SetClientSize(wx.DisplaySize())
# bitmap's upper left corner is in frame position (x, y)
# by default pos=(0, 0)
wx.StaticBitmap(self, wx.ID_ANY, image, size = image_size)
# the parent is the frame
self.SetTitle('Update Mode')
def ping_ip():
""" ping once to establish connection """
ret = subprocess.call("ping -c 1 %s" % PING_IP,
shell=True,
stdout=open('/dev/null', 'w'),
stderr=subprocess.STDOUT)
if ret == 0:
return True
else:
return False
def show_window():
""" update screen window when currimage changes is set """
global UPDATE_UI
global WINDOW
global CURR_IMG
global LAST_CURRIMG
if LAST_CURRIMG != CURR_IMG:
if not UPDATE_UI:
UPDATE_UI = wx.App()
if not WINDOW:
WINDOW = UpdateFrame(CURR_IMG)
UPDATE_UI.ExitMainLoop()
while(UPDATE_UI.IsMainLoopRunning()):
pass
WINDOW.Destroy()
WINDOW = UpdateFrame(CURR_IMG)
WINDOW.Show(True)
UPDATE_UI.MainLoop()
LAST_CURRIMG = CURR_IMG
print 'changed'
def in_updatemode():
return ping_ip()
while True:
try:
if not in_updatemode():
print 'waiting for connect'
time.sleep(3)
if NOTIFIER:
NOTIFIER.stop()
else:
if not HANDLER:
HANDLER = EventHandler()
if not NOTIFIER:
NOTIFIER = pyinotify.ThreadedNotifier(WM, HANDLER)
NOTIFIER.start()
if not WDD:
WDD = WM.add_watch(WATCH_PATH, WM_MASK, rec=True,quiet=False)
# ip is active so show the image and start the notifier
# state = ip active
CURR_IMG = os.path.join(WATCH_PATH, CONNECTED_IMG)
show_window()
print 'here'
except KeyboardInterrupt:
print 'out'
NOTIFIER.stop()
break

I basically found out that that the issue was indeed the fact that pyinotify's threaded notifier and wxPython's main loop dont play nice together.
The solution was to create a custom main loop (something I didn't know you could do in the first place) and place the pyinotify's threaded notifier within that loop. That way it ran as part of the wxPython main loop.
I got the idea from
http://www.java2s.com/Open-Source/Python/GUI/wxPython/wxPython-src-2.8.11.0/wxPython/samples/mainloop/mainloop.py.htm
Code below explains the concept
class CustomApp(wx.App):
def MainLoop(self):
global HANDLER
global WM
global NOTIFIER
global WDD
global UPDATE_UI
global PING_TIMER
# Create an event loop and make it active. If you are
# only going to temporarily have a nested event loop then
# you should get a reference to the old one and set it as
# the active event loop when you are done with this one...
evtloop = wx.EventLoop()
old = wx.EventLoop.GetActive()
wx.EventLoop.SetActive(evtloop)
# This outer loop determines when to exit the application,
# for this example we let the main frame reset this flag
# when it closes.
while self.keepGoing:
# At this point in the outer loop you could do
# whatever you implemented your own MainLoop for. It
# should be quick and non-blocking, otherwise your GUI
# will freeze.
# call_your_code_here()
if not HANDLER:
HANDLER = EventHandler()
if not WM:
WM = pyinotify.WatchManager() # Watch Manager
if not NOTIFIER:
NOTIFIER = pyinotify.ThreadedNotifier(WM, HANDLER)
NOTIFIER.start()
print 'notifier started'
if not WDD:
WDD = WM.add_watch(WATCH_PATH, WM_MASK, rec=True,quiet=False)
# This inner loop will process any GUI events
# until there are no more waiting.
while evtloop.Pending():
evtloop.Dispatch()
# Send idle events to idle handlers. You may want to
# throttle this back a bit somehow so there is not too
# much CPU time spent in the idle handlers. For this
# example, I'll just snooze a little...
time.sleep(0.10)
self.ProcessIdle()
wx.EventLoop.SetActive(old)
def OnInit(self):
global UPDATE_UI
if not UPDATE_UI:
UPDATE_UI = Updater()
UPDATE_UI.Show()
self.SetTopWindow(UPDATE_UI)
self.keepGoing = True
return True
"--------------------------------------------------------------------------------------"
Watcher()
app = CustomApp(False)
Additionally, I wanted to catch SIGINT in the program and I solved it using the recipe from this url
http://code.activestate.com/recipes/496735-workaround-for-missed-sigint-in-multithreaded-prog/
Hope this helps another python newbie or oldie :)