terminal partitioning for each of subprocesses prints - python

Say we have multiple subprocesses like the following which has some results printed in real time to sys.stdout or sys.stderr.
proc1 = subprocess.Popen(['cmd1'],
env=venv1,
stdout=sys.stdout,
stderr=sys.stderr,
)
proc2 = subprocess.Popen(['cmd2'],
env=venv2,
stdout=sys.stdout,
stderr=sys.stderr,
)
However, after executing this script in the terminal, while looking at what is being printed, it is not easy to distinguish which print is from the first process and which is from the second.
Is there a solution for this to see the stdout of each process separately, like if the terminal screen could be partitioned and each partition would have shown the printing results from each process?

I have written for you a curses application which will do what you request: divide the terminal window into a number of partitions and then watch the different output streams in the different partitions.
The function watch_fd_in_panes will take a list of lists, where the sub-lists specify which file descriptors to watch inside each partition.
Here is what your example calling code will look like:
import subprocess
from watcher import watch_fds_in_panes
proc1 = subprocess.Popen('for i in `seq 30`; do date; sleep 1 ; done',
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
# this process also writes something on stderr
proc2 = subprocess.Popen('ls -l /asdf; for i in `seq 20`; do echo $i; sleep 0.5; done',
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
proc3 = subprocess.Popen(['echo', 'hello'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
try:
watch_fds_in_panes([[proc1.stdout.fileno(), proc1.stderr.fileno()],
[proc2.stdout.fileno(), proc2.stderr.fileno()],
[proc3.stdout.fileno(), proc3.stderr.fileno()]],
sleep_at_end=3.)
except KeyboardInterrupt:
print("interrupted")
proc1.kill()
proc2.kill()
proc3.kill()
To run it you will need these two files:
panes.py
import curses
class Panes:
"""
curses-based app that divides the screen into a number of scrollable
panes and lets the caller write text into them
"""
def start(self, num_panes):
"set up the panes and initialise the app"
# curses init
self.num = num_panes
self.stdscr = curses.initscr()
curses.noecho()
curses.cbreak()
# split the screen into number of panes stacked vertically,
# drawing some horizontal separator lines
scr_height, scr_width = self.stdscr.getmaxyx()
div_ys = [scr_height * i // self.num for i in range(1, self.num)]
for y in div_ys:
self.stdscr.addstr(y, 0, '-' * scr_width)
self.stdscr.refresh()
# 'boundaries' contains y coords of separator lines including notional
# separator lines above and below everything, and then the panes
# occupy the spaces between these
boundaries = [-1] + div_ys + [scr_height]
self.panes = []
for i in range(self.num):
top = boundaries[i] + 1
bottom = boundaries[i + 1] - 1
height = bottom - top + 1
width = scr_width
# create a scrollable pad for this pane, of height at least
# 'height' (could be more to retain some scrollback history)
pad = curses.newpad(height, width)
pad.scrollok(True)
self.panes.append({'pad': pad,
'coords': [top, 0, bottom, width],
'height': height})
def write(self, pane_num, text):
"write text to the specified pane number (from 0 to num_panes-1)"
pane = self.panes[pane_num]
pad = pane['pad']
y, x = pad.getyx()
pad.addstr(y, x, text)
y, x = pad.getyx()
view_top = max(y - pane['height'], 0)
pad.refresh(view_top, 0, *pane['coords'])
def end(self):
"restore the original terminal behaviour"
curses.nocbreak()
self.stdscr.keypad(0)
curses.echo()
curses.endwin()
and watcher.py
import os
import select
import time
from panes import Panes
def watch_fds_in_panes(fds_by_pane, sleep_at_end=0):
"""
Use panes to watch output from a number of fds that are writing data.
fds_by_pane contains a list of lists of fds to watch in each pane.
"""
panes = Panes()
npane = len(fds_by_pane)
panes.start(npane)
pane_num_for_fd = {}
active_fds = []
data_tmpl = {}
for pane_num, pane_fds in enumerate(fds_by_pane):
for fd in pane_fds:
active_fds.append(fd)
pane_num_for_fd[fd] = pane_num
data_tmpl[fd] = bytes()
try:
while active_fds:
all_data = data_tmpl.copy()
timeout = None
while True:
fds_read, _, _ = select.select(active_fds, [], [], timeout)
timeout = 0
if fds_read:
for fd in fds_read:
data = os.read(fd, 1)
if data:
all_data[fd] += data
else:
active_fds.remove(fd) # saw EOF
else:
# no more data ready to read
break
for fd, data in all_data.items():
if data:
strng = data.decode('utf-8')
panes.write(pane_num_for_fd[fd], strng)
except KeyboardInterrupt:
panes.end()
raise
time.sleep(sleep_at_end)
panes.end()
Finally, here is a screenshot of the above code in action:
In this example, we are monitoring both stdout and stderr of each process in the relevant partition. In the screenshot, the line that proc2 wrote to stderr before the start of the loop (regarding /asdf) has appeared after the first line that proc2 wrote to stdout during the first iteration of the loop (i.e. the 1 which has since scrolled off the top of the partition), but this is cannot be controlled because they were written to different pipes.

Related

I want to create with pyhtonsimple gui a screen with a working terminal how do I do?

I want to create with pyhtonsimple gui a screen with a working terminal how do I do?
the starting program I am using is this:
import subprocess
import sys
import PySimpleGUI as sg
sg.theme('DarkAmber') # Add a touch of color
All the stuff inside your window.
layout = [ [sg.Text('Terminal')],
[sg.Output(size=(60,15))], # an output area where all print output will go
[sg.Text('Writhe her'), sg.InputText()],
[sg.Button('Send'), sg.Button('EXIT')] ]
Create the Window
window = sg.Window('Sedia', layout)
Event Loop to process "events" and get the "values" of the inputs
while True:
event, values = window.read()
if (event == sg.WINDOW_CLOSE_ATTEMPTED_EVENT or event == 'EXIT') and sg.popup_yes_no('Do you really want to EXIT?') == 'Yes':
break
print('', values[0])
def runCommand(cmd, timeout=None, window=None):
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = ''
for line in p.stdout:
line = line.decode(errors='replace' if (sys.version_info) < (3, 5) else 'backslashreplace').rstrip()
output += line
print(line)
window.Refresh() if window else None # yes, a 1-line if, so shoot me
retval = p.wait(timeout)
return (retval, output) # also return the output just for fun
window.close()
instead of the text file that opens I would like the terminal to open.

Multiprocessing Pipe send() blocks

According to Python documentation, only recv() blocks but not send(). I wrote the following code trying to make a GUI sudoku game. I made it in such a way that I can update the game board even if the tkinter is executing its mainloop. However, during a test run, I found that if I close the window while the game is updating, the pipe.send() starts to block (I found that out using CPython profiler.) Can anyone please tell me why and, if possible, how to fix this issue?
To produce the issue: close the poped up window while the script is updating. That is, close the window while some numbers are printed to console.
My system: macOS Sierra 10.12.5
import multiprocessing as mp
import threading
import random
import time
try:
import tkinter as tk # Python3
except ImportError:
import Tkinter as tk # Python2
class VisualizedBoard:
def __init__(self,input_string,pipe):
'''input_string: a string has a length of at least 81 that represent the board from top-left to bottom right.
empty cell is 0'''
self.update_scheduled=False
self.pipe=pipe
# create board
self.root = tk.Tk()
self.canvas = tk.Canvas(self.root, width=500, height=500)
self.canvas.create_rectangle(50, 50, 500, 500, width=2)
for i in range(1, 10):
self.canvas.create_text(25 + 50 * i, 30, text=str(i))
self.canvas.create_text(30, 25 + 50 * i, text=str(i))
self.canvas.create_line(50 + 50 * i, 50, 50 + 50 * i, 500, width=2 if i % 3 == 0 else 1)
self.canvas.create_line(50, 50 + 50 * i, 500, 50 + 50 * i, width=2 if i % 3 == 0 else 1)
for i in range(81):
if input_string[i] != '0':
self.canvas.create_text(75 + 50 * (i // 9), 75 + 50 * (i % 9), tags=str((i//9+1,i%9+1)).replace(' ',''),text=input_string[i], fill='black')
self.canvas.pack()
self.root.attributes('-topmost', True)
self.root.geometry('550x550+%d+%d' % ((self.root.winfo_screenwidth() - 550) // 2, (self.root.winfo_screenheight() - 550) // 2))
self.root.wm_protocol('WM_DELETE_WINDOW',lambda :(self.root.destroy()))
threading.Thread(target=self.listen, args=()).start()
self.root.mainloop()
def update(self,coordinate,value,color='magenta'):
"""
:parameter coordinate: a tuple (x,y)
:parameter value: single digit
:returns: None
"""
try:
assert isinstance(coordinate,tuple)
except AssertionError:
print('Update Failed. Coordinate should be a tuple (x,y)')
coordinate_tag=str(coordinate).replace(' ','')
self.canvas.delete(coordinate_tag)
if value != 0 and value != '0':
self.canvas.create_text(25+50*coordinate[0],25+50*coordinate[1],tags=coordinate_tag,text=str(value),fill=color)
self.postponed_update()
#self.canvas.update()
def postponed_update(self):
if not self.update_scheduled:
self.canvas.after(50,self.scheduled_update)
self.update_scheduled=True
def scheduled_update(self):
self.canvas.update()
self.update_scheduled=False
def new_board(self,input_string):
self.root.destroy()
return VisualizedBoard(input_string,self.pipe)
def listen(self):
try:
while True:
msg=self.pipe.recv()
self.update(*msg)
except EOFError:
self.pipe.close()
tk.Label(self.root,text='Connection to the main script has been closed.\nIt is safe to close this window now.').pack()
except Exception as m:
self.pipe.close()
print('Error during listing:',m)
class BoardConnection:
def __init__(self,input_string):
ctx = mp.get_context('spawn')
self.receive,self.pipe=ctx.Pipe(False)
self.process=ctx.Process(target=VisualizedBoard,args=(input_string,self.receive))
self.process.start()
def update(self,coordinate,value,color='magenta'):
"""
:parameter coordinate: a tuple (x,y)
:parameter value: single digit
:returns: None
"""
self.pipe.send((coordinate,value,color))
def close(self):
self.pipe.close()
self.process.terminate()
if __name__ == "__main__":
b=BoardConnection('000000000302540000050301070000000004409006005023054790000000050700810000080060009')
start = time.time()
for i in range(5000): #test updating using random numbers
b.update((random.randint(1, 9), random.randint(1, 9)), random.randrange(10))
print(i)
print(time.time() - start)
Python Pipe is an abstractions on top of OS nameless pipes.
An OS pipe is generally implemented as a memory buffer of a certain size within the kernel. By default, if the buffer fills up the next call to send/write will block.
If you want to be able to continue publishing data even if no consumer is consuming it, you should either use a multiprocessing.Queue or asyncio facilities.
The multiprocessing.Queue employs a "limitless" buffer and a thread to push the data into the OS pipe. If the pipe gets full the caller will continue running as the published data will be piled up in the Queue buffer.
IIRC, asyncio sets the pipe O_NONBLOCK flag and waits for the pipe to be consumed. Additional messages are stored within a "limitless" buffer as for the multiprocessing.Queue.

Why is the Windows prompt unresponsive after using a python curses window?

First of all, I'm a newby in python. Had to take a course of it in college and got hooked by its efficiency.
I have this sticky problem where the Windows 7 prompt becomes unresponsive after using a curses window. In Windows 10 it works well. Note that I'm using the Win7 terminal with its default settings. In my code I create a curses window to show 2 simultaneous progress bars, each for a file download. I implemented this by passing the curses window to a FileDownload class (one class instance for each download) that handles its progress bar inside this window. Oddly, in Windows 7 when the downloads are done and the control returns to the prompt, it becomes unresponsive to the keyboard. I worked around this by invoking curses.endwin() after using the window, but this causes the prompt to display all the way down the screen buffer, what hides the curses window.
Here is my code. Any ideas are greatly appreciated. Thanks!
# Skeleton version for simulations.
# Downloads 2 files simultaneously and shows a progress bar for each.
# Each file download is a FileDownload object that interacts with a
# common curses window passed as an argument.
import requests, math, threading, curses, datetime
class FileDownload:
def __init__(self, y_pos, window, url):
# Y position of the progress bar in the download queue window.
self.__bar_pos = int(y_pos)
self.__progress_window = window
self.__download_url = url
# Status of the file download object.
self.__status = "queued"
t = threading.Thread(target=self.__file_downloader)
t.start()
# Downloads selected file and handles its progress bar.
def __file_downloader(self):
file = requests.get(self.__download_url, stream=True)
self.__status = "downloading"
self.__progress_window.addstr(self.__bar_pos + 1, 1, "0%" + " " * 60 + "100%")
size = int(file.headers.get('content-length'))
win_prompt = "Downloading " + format(size, ",d") + " Bytes:"
self.__progress_window.addstr(self.__bar_pos, 1, win_prompt)
file_name = str(datetime.datetime.now().strftime("%Y-%m-%d_%H.%M.%d"))
dump = open(file_name, "wb")
# Progress bar length.
bar_space = 58
# Same as an index.
current_iteration = 0
# Beginning position of the progress bar.
progress_position = 4
# How many iterations will be needed (in chunks of 1 MB).
iterations = math.ceil(size / 1024 ** 2)
# Downloads the file in 1MB chunks.
for block in file.iter_content(1024 ** 2):
dump.write(block)
# Progress bar controller.
current_iteration += 1
step = math.floor(bar_space / iterations)
if current_iteration > 1:
progress_position += step
if current_iteration == iterations:
step = bar_space - step * (current_iteration - 1)
# Updates the progress bar.
self.__progress_window.addstr(self.__bar_pos + 1, progress_position,
"#" * step)
dump.close()
self.__status = "downloaded"
# Returns the current status of the file download ("queued", "downloading" or
# "downloaded").
def get_status(self):
return self.__status
# Instantiates each file download.
def files_downloader():
# Creates curses window.
curses.initscr()
win = curses.newwin(8, 70)
win.border(0)
win.immedok(True)
# Download URLs.
urls = ["http://ipv4.download.thinkbroadband.com/10MB.zip",
"http://ipv4.download.thinkbroadband.com/5MB.zip"]
downloads_dct = {}
for n in range(len(urls)):
# Progress bar position in the window for the file.
y_pos = n * 4 + 1
downloads_dct[n + 1] = FileDownload(y_pos, win, urls[n])
# Waits for all files to be downloaded before passing control of the terminal
# to the user.
all_downloaded = False
while not all_downloaded:
all_downloaded = True
for key, file_download in downloads_dct.items():
if file_download.get_status() != "downloaded":
all_downloaded = False
# Prevents the prompt from returning inside the curses window.
win.addstr(7, 1, "-")
# This solves the unresponsive prompt issue but hides the curses window if the screen buffer
# is higher than the window size.
# curses.endwin()
while input("\nEnter to continue: ") == "":
files_downloader()
Perhaps you're using cygwin (and ncurses): ncurses (like any other curses implementation) changes the terminal I/O mode when it is running. The changes that you probably are seeing is that
input characters are not echoed
you have to type controlJ to end an input line, rather than just Enter
output is not flushed automatically at the end of each line
It makes those changes to allow it to read single characters and also to use the terminal more efficiently.
To change back to the terminal's normal I/O mode, you would use the endwin function. The reset_shell_mode function also would be useful.
Further reading:
endwin (ncurses manual)
reset_shell_mode (ncurses manual)

Print 2 lines in the console concurrently in Python

I'm using Python 3 to output 2 progress bars in the console like this:
100%|###############################################|
45%|###################### |
Both bars grow concurrently in separate threads.
The thread operations are fine and both progress bars are doing their job, but when I want to print them out they print on top of each other on one line in the console. I just got one line progress bar which alternates between showing these 2 progress bars.
Is there any way these progress bars can grow on separate lines concurrently?
You need a CLI framework. Curses is perfect if you are working on Unix (and there is a port for Windows which can be found here : https://stackoverflow.com/a/19851287/1741450 )
import curses
import time
import threading
def show_progress(win,X_line,sleeping_time):
# This is to move the progress bar per iteration.
pos = 10
# Random number I chose for demonstration.
for i in range(15):
# Add '.' for each iteration.
win.addstr(X_line,pos,".")
# Refresh or we'll never see it.
win.refresh()
# Here is where you can customize for data/percentage.
time.sleep(sleeping_time)
# Need to move up or we'll just redraw the same cell!
pos += 1
# Current text: Progress ............... Done!
win.addstr(X_line,26,"Done!")
# Gotta show our changes.
win.refresh()
# Without this the bar fades too quickly for this example.
time.sleep(0.5)
def show_progress_A(win):
show_progress( win, 1, 0.1)
def show_progress_B(win):
show_progress( win, 4 , 0.5)
if __name__ == '__main__':
curses.initscr()
win = curses.newwin(6,32,14,10)
win.border(0)
win.addstr(1,1,"Progress ")
win.addstr(4,1,"Progress ")
win.refresh()
threading.Thread( target = show_progress_B, args = (win,) ).start()
time.sleep(2.0)
threading.Thread( target = show_progress_A, args = (win,)).start()

Concurrent functions running in separate process using pygame and multiprocessing

Suppose we want to drive an autonomous car by predicting image labels from a previous set of images and labels collected (A Machine Learning application). For this task, the car is connected via bluetooth serial (rfcomm) to the Host Computer (A PC with *NIX) and the images are streamed directly from an Android phone using IP Webcam, meanwhile, the PC is running a program that links this two functions, displaying the captured images in a drawing environment created by pygame, and sending the instructions back to the car using serial.
At the moment, I've tried to implement those processes using the multiprocessing module, the seemed to work, but when I execute the client, the drawing function (if __name__ == '__main__') works after the getKeyPress() function ends.
The question is: It is possible to parallelize or synchronize the drawing fuinction enclosed within the if __name__ == '__main__' with the process declared in getKyPress(), such that the program works in two independent processes?
Here's the implemented code so far:
import urllib
import time
import os
import sys
import serial
import signal
import multiprocessing
import numpy as np
import scipy
import scipy.io as sio
import matplotlib.image as mpimg
from pygame.locals import *
PORT = '/dev/rfcomm0'
SPEED = 115200
ser = serial.Serial(PORT)
status = False
move = None
targets = []
inputs = []
tic = False
def getKeyPress():
import pygame
pygame.init()
global targets
global status
while not status:
pygame.event.pump()
keys = pygame.key.get_pressed()
targets, status = processOutputs(targets, keys)
targets = np.array(targets)
targets = flattenMatrix(targets)
sio.savemat('targets.mat', {'targets':targets})
def rgb2gray(rgb):
r, g, b = np.rollaxis(rgb[...,:3], axis = -1)
return 0.299 * r + 0.587 * g + 0.114 * b
def processImages(inputX, inputs):
inputX = flattenMatrix(inputX)
if len(inputs) == 0:
inputs = inputX
elif inputs.shape[1] >= 1:
inputs = np.hstack((inputs, inputX))
return inputs
def flattenMatrix(mat):
mat = mat.flatten(1)
mat = mat.reshape((len(mat), 1))
return mat
def send_command(val):
connection = serial.Serial( PORT,
SPEED,
timeout=0,
stopbits=serial.STOPBITS_TWO
)
connection.write(val)
connection.close()
def processOutputs(targets, keys):
global move
global status
global tic
status = False
keypress = ['K_p', 'K_UP', 'K_LEFT', 'K_DOWN', 'K_RIGHT']
labels = [1, 2, 3, 4, 5]
commands = ['p', 'w', 'r', 'j', 's']
text = ['S', 'Up', 'Left', 'Down', 'Right']
if keys[K_q]:
status = True
return targets, status
else:
for i, j, k, g in zip(keypress, labels, commands, text):
cmd = compile('cond = keys['+i+']', '<string>', 'exec')
exec cmd
if cond:
move = g
targets.append(j)
send_command(k)
break
send_command('p')
return targets, status
targetProcess = multiprocessing.Process(target=getKeyPress)
targetProcess.daemon = True
targetProcess.start()
if __name__ == '__main__':
import pygame
pygame.init()
w = 288
h = 352
size=(w,h)
screen = pygame.display.set_mode(size)
c = pygame.time.Clock() # create a clock object for timing
pygame.display.set_caption('Driver')
ubuntu = pygame.font.match_font('Ubuntu')
font = pygame.font.Font(ubuntu, 13)
inputs = []
try:
while not status:
urllib.urlretrieve("http://192.168.0.10:8080/shot.jpg", "input.jpg")
try:
inputX = mpimg.imread('input.jpg')
except IOError:
status = True
inputX = rgb2gray(inputX)/255
out = inputX.copy()
out = scipy.misc.imresize(out, (352, 288), interp='bicubic', mode=None)
scipy.misc.imsave('input.png', out)
inputs = processImages(inputX, inputs)
print inputs.shape[1]
img=pygame.image.load('input.png')
screen.blit(img,(0,0))
pygame.display.flip()
c.tick(1)
if move != None:
text = font.render(move, False, (255, 128, 255), (0, 0, 0))
textRect = text.get_rect()
textRect.centerx = 20 #screen.get_rect().centerx
textRect.centery = 20 #screen.get_rect().centery
screen.blit(text, textRect)
pygame.display.update()
if status:
targetProcess.join()
sio.savemat('inputs.mat', {'inputs':inputs})
except KeyboardInterrupt:
targetProcess.join()
sio.savemat('inputs.mat', {'inputs':inputs})
targetProcess.join()
sio.savemat('inputs.mat', {'inputs':inputs})
Thanks in advance.
I would personally suggest writing this without using the multiprocessing module: it uses fork() which has unspecified effects with most complex libraries, like in this case pygame.
You should try to write this as two completely separate programs. It forces you to think about what data needs to go from one to the other, which is both a bad and a good thing (as it may clarify things). You can use some inter-process communication facility, like the stdin/stdout pipe; e.g. in one program (the "main" one) you start the other as a sub-process like this:
popen = subprocess.Popen([sys.executable, '-u', 'my_subproc.py'],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
(The -u is for unbuffered.)
Then read/write the data to popen.stdin/popen.stdout in the parent process, and to sys.stdin/sys.stdout in the subprocess. The simplest example would be if the two processes only need a synchronization signal, e.g. the parent process waits in a loop for the subprocess to say "next please". To do this the subprocess does print 'next please', and the parent process does popen.stdin.readline(). (The print goes to sys.stdin in the subprocess.)
Unrelated small note:
keypress = ['K_p', ...]
...
cmd = compile('cond = keys['+i+']', '<string>', 'exec')
exec cmd
if cond:
This looks like very heavy code to just do:
keypress = [K_p, ...] # not strings, directly the values
...
if keys[i]:
My suggestion is to use separate threads.
#At the beginning
import threading
#Instead of def getKeyPress()
class getKeyPress(threading.Thread):
def run(self):
import pygame
pygame.init()
global targets
global status
while not status:
pygame.event.pump()
keys = pygame.key.get_pressed()
targets, status = processOutputs(targets, keys)
targets = np.array(targets)
targets = flattenMatrix(targets)
sio.savemat('targets.mat', {'targets':targets})
#Instead of
#targetProcess = multiprocessing.Process(target=getKeyPress)
#targetProcess.daemon = True
#targetProcess.start()
gkp = getKeyPress()
gkp.start()
An alternative would be creating two different scripts and using sockets to handle the inter-process communication.

Categories

Resources