I am trying to learn how to write interactive subprocess communication.
I need to read stdout from and write to stdin continually, below is my code, it sort of "works" but I am not sure if I am doing it right (it's very hacked code)
Assuming I have a script called app.py as following
import logging
import random
def app():
number1 = random.randint(1,100)
number2 = random.randint(200,500)
logging.info("number1: %s, number2: %s", number1, number2)
ans = input("enter sum of {} and {}: ".format(number1, number2))
logging.info("received answer: %s", ans)
try:
if int(ans) != number1+number2:
raise ValueError
logging.info("{} is the correct answer".format(ans))
except (ValueError,TypeError):
logging.info("{} is incorrect answer".format(ans))
def main():
logging.basicConfig(level=logging.DEBUG, filename='log.log')
for x in range(10):
app()
if __name__ == '__main__':
main()
to interactive with above script (app.py) I have some very ugly code
import queue
import time
import threading
import subprocess
import os
import pty
import re
class ReadStdout(object):
def __init__(self):
self.queue = queue.Queue()
self._buffer_ = []
def timer(self, timeout=0.1):
buffer_size = 0
while True:
if len(self._buffer_) > buffer_size:
buffer_size = len(self._buffer_)
time.sleep(timeout)
if len(self._buffer_) == buffer_size and buffer_size!=0:
self.queue.put(''.join(self._buffer_))
self._buffer_ = []
buffer_size = 0
def read(self, fd):
while True:
self._buffer_.append(fd.read(1))
def run(self):
timer = threading.Thread(target=self.timer)
timer.start()
master, slave = pty.openpty()
p = subprocess.Popen(['python', 'app.py'], stdout=slave, stderr=slave, stdin=subprocess.PIPE, close_fds=True)
stdout = os.fdopen(master)
read_thread = threading.Thread(target=self.read, args=(stdout,))
read_thread.start()
while True:
if self.queue.empty():
time.sleep(0.1)
continue
msg = self.queue.get()
digits = (re.findall('(\d+)', msg))
ans = (int(digits[0])+int(digits[1]))
print("got message: {} result: {}".format(msg, ans))
p.stdin.write(b"%d\n" %ans)
p.stdin.flush()
if __name__ == '__main__':
x = ReadStdout()
x.run()
I don't feel I am doing it the right way. what's the correct way to interactive with another script (I need stdout, not just blind write to stdin)
Thanks
This code will work with your app.py, so you can get base logic of interaction from it. Also, i would suggest you to look into pexpect module. In any case - you MUST know what to expect from running program and how it's input lines endswith. Or you could implement some timeout while reading line, so it would be possible to raise exception if something went not as expected.
import subprocess
from functools import partial
child = subprocess.Popen(['app.py'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True)
# iterate while child is not terminated
while child.poll() is None:
line = ''
# read stdout character by character until a colon appears
for c in iter(partial(child.stdout.read, 1), ''):
if c == ':':
break
line += c
if "enter sum" in line:
numbers = filter(str.isdigit, line.split())
numbers = list(map(int, numbers))
child.stdin.write("{0}\n".format(sum(numbers)))
Related
I want to create a while loop that is able to keep on receiving inputs but will break once a timer countdown to 0. How am I able to do it?
while True:
x = raw_input("Enter Stuff here:")
This loop will end once timer reaches zero, regardless if the user has anything inputted or not. However, i do not know how to implement such timer
I've done something similar previously, where I use signal to trigger an error when the time I've allotted has run out, basically a TimeOutError. I have it as a function wrapper which I can conveniently use it as #timeout(seconds=<number_of_seconds>). See the code below:
import signal
from functools import wraps
class TimeOutError(Exception):
pass
def timeout(seconds):
def decorator(func):
def _handle_timeout(signum, frame):
raise TimeOutError()
def wrapper(*args, **kwargs):
signal.signal(signal.SIGALRM, _handle_timeout)
signal.alarm(seconds)
try:
result = func(*args, **kwargs)
finally:
signal.alarm(0)
return result
return wraps(func)(wrapper)
return decorator
# Using it as a function wrapper
#timeout(seconds=10)
def run():
while True: x = raw_input("Enter Stuff here:")
# Then calling it as such
try:
run()
except TimeOutError:
print("You have exceeded the allotted time!")
import sys
import threading
import time
import queue
def add_input(input_queue):
while True:
input_queue.put(sys.stdin.read(1))
def timed_input():
s = ""
input_queue = queue.Queue()
input_thread = threading.Thread(target=add_input, args=(input_queue,))
input_thread.daemon = True
input_thread.start()
last_update = time.time()
print("Enter stuff here and confirm with enter:")
while True:
# Do while countdown of 5 seconds
if time.time()-last_update > 5:
print("\nTime is up")
s = s.replace("\n", "")
if s != "":
print(s, " entered.")
break
if not input_queue.empty():
s += input_queue.get()
timed_input()
You can do it by using multiprocessing.Process
from time import sleep
from os.path import isfile
from os import remove
def timeout():
t_start = time()
while True:
if time() - t_start > 3:
with open('spy', 'w') as source:
source.write('emm')
break
else:
sleep(1.)
p1 = Process(target=timeout, daemon=True)
p1.start()
while True:
if not isfile('spy'):
a = input('input something:')
else:
remove('spy')
p1.join()
break
However, as you can see from the script, one problem is this script won't exit unless a user types something to invoke if not isfile('spy').
I have a program like this:
from threading import Thread
def foo1(arg):
print("foo1 >>> Something")
input("foo1 >>> Enter Something")
...
def foo2(arg):
print("foo2 >>> Something")
input("foo2 >>> Enter Something")
...
def main():
th1 = Thread(target= foo1)
th1.start()
th2 = Thread(target= foo2)
th2.start()
This program runs both the functions(foo1 and foo2) in the same terminal window. Can I in some way run them in a different terminal window. What I don't wish is to re-run the program. The reason is that they print and take input at the same place and same time. I don't want. Any method?
What you are trying to accomplish isn't possible with just threads, when you create a new Thread it shares all the variables with other threads in your program, including sys.stdout / sys.stdin.
Normally you don't have to worry about PIPES in python programs because it takes care of it for you. print sends the text to sys.stdout and input grabs text from sys.stdin (and error messages are sent to sys.stderr)
So running one program in two terminal windows would mean you would have to have more then one input/output streams, to which there are two solutions:
run a completely separate program with subprocess.Popen like the other fellow described and figure out how to bridge information across the two which is a real pain.
or 2. create your own terminal window with something like tkinter, which is difficult from scratch but luckily IDLE has the majority of the code available in the standard library.
Here is an adapted version of PyShell from idlelib.PyShell to run a Thread instead of the interactive interpretive:
from idlelib import PyShell,EditorWindow
import threading,sys
try:
import tkinter as tk #python 3.X
except ImportError:
import Tkinter as tk #python 2
import tkMessageBox as messagebox
tk.messagebox = messagebox
class ThreadShell(PyShell.PyShell):
"""mostly copied from idlelib.PyShell module but adapted to work with threads"""
#__adapted_by__ = "Tadhg McDonald-Jensen"
def __init__(self, tk_root,target=None):
#not sure exactly what the FileList object is for but it is required by the shell
flist = PyShell.PyShellFileList(tk_root)
super(ThreadShell,self).__init__(flist)
#internal event flag for input, allows thread waiting for input to wait until a tk event handles it
self.__input_flag = threading.Event()
#target is stored and called in .run_command() which also deals with finishing the shell
self.target = target
self.thread = threading.Thread(target=self.run_command)
#tk_root.after makes the .start method call when the program starts (after 0 miliseconds)
tk_root.after(0,self.start)
def start(self):
"""starts executing the Thread"""
super(ThreadShell,self).beginexecuting()
try:
self.thread.start()
except RuntimeError:
self.executing = 0
self.canceled = 0
#self.top.quit() #this causes double deletion warnings with better Implementation of mainloop
beginexecuting = start
def run_command(self):
"""calls target from constructor with self as argument then cleans up shell"""
if self.target:
self.target(self)
self.prompt_exit()
self.executing = 0
self.canceled = 0
try:
self.text.after(1,self.close)
except RuntimeError:
pass #tkinter has issues with changing threads so often after closing one shell others will throw this error
def printf(self,*stuff,**kw):
"""works just like python 3.x print function but writes to shell's .stdout file"""
if self.executing:
## if USING_OLD_METHOD: #Pretty sure this would do exact same thing
## kw.setdefault("file",self.stdout)
## print(*stuff,**kw), self.resetoutput()
## return
sep = kw.get("sep"," ")
end = kw.get("end","\n")
text = sep.join(stuff) + end
self.stdout.write(text)
self.resetoutput()
def input(self,prompt="",timeout=None):
"""python 2 equivelent to raw_input or py 3+ input
Prompts user for input and freezes thread until input is given
Will return "" if .executing is False or it timed out from optional timeout argument"""
if self.executing or self.closing:
if prompt:
self.stdout.write(prompt)
self.__in_buffer = ""
self.__input_flag.clear()
self.reading=True
self.__input_flag.wait(timeout)
#input is inserted into .__in_buffer by other events
#then set __input_flag so that it can be delivered to thread
self.reading = False
return self.__in_buffer.strip("\n")
else:
raise RuntimeError("cannot take input after finished")
def prompt_exit(self):
"""writes press enter to quit" to the console colour then waits for input"""
self.executing = False
self.closing = True
self.console.write("\n press enter to quit")
self.input()
def join_thread(self,timeout=None):
"""sets .executing label to False then waits to join thead,
returns True if thread finished or False if timeout activated"""
self.executing = False
self.closing = True
if self.thread:
self.thread.join(timeout)
return not self.thread.is_alive()
def _close(self):
"Extend EditorWindow._close(), joins thread to close it"
# Restore std streams
sys.stdout = self.save_stdout
sys.stderr = self.save_stderr
sys.stdin = self.save_stdin
# Break cycles
self.interp = None
self.console = None
self.flist.pyshell = None
self.history = None
EditorWindow.EditorWindow._close(self)
self.join_thread()
def stop_readline(self):
self.__in_buffer = ""
self.__input_flag.set()
def update_in(self):
"""updates input from user, I think some of the labels are probably unnecessary but it is easier to leave it alone"""
line = self.text.get("iomark", "end-1c")
if len(line) == 0: # may be EOF if we quit our mainloop with Ctrl-C
line = "\n"
self.resetoutput()
if self.canceled:
self.canceled = 0
if self.endoffile:
self.endoffile = 0
line = ""
self.__in_buffer = line
self.__input_flag.set()
def cancel_callback(self, event=None):
try:
if self.text.compare("sel.first", "!=", "sel.last"):
return # Active selection -- always use default binding
except:
pass
if not (self.executing or self.reading):
return "break"
self.endoffile = 0
self.canceled = 1
if self.reading:
self.update_in()
return "break"
def eof_callback(self, event):
if self.executing and not self.reading:
return # Let the default binding (delete next char) take over
if not (self.text.compare("iomark", "==", "insert") and
self.text.compare("insert", "==", "end-1c")):
return # Let the default binding (delete next char) take over
if not self.executing:
self.resetoutput()
self.close()
else:
self.canceled = 0
self.endoffile = 1
self.update_in()
return "break"
def enter_callback(self, event):
"""called when the enter/return key is pressed,
only the recursive self.top.mainloop() / self.top.quit() had to be changed for support"""
# it is very long to copy/paste for the one line change, so I override the method temporarily
save = self.top.quit
self.top.quit = self.update_in
super(ThreadShell,self).enter_callback(event)
self.top.quit = save
#stupid module depends on this being set from the main function, so it needs to be done manually
PyShell.use_subprocess = True
#this defines the root tkinter window and sets it up
root = tk.Tk()
EditorWindow.fixwordbreaks(root)
root.withdraw()
#I need this to work on my mac, not sure if there are other OS specific stuff that should be included
try:
from idlelib import macosxSupport
macosxSupport.setupApp(root, None)
except (ImportError,AttributeError):
pass
##!!!!!!!!!!!!!!!!!!!! And This Is The Part You Need To Worry About !!!!!!!!!!!!!!!!!!!!##
switch = threading.Event()
switch.clear()
def foo(shell):
global x
x = shell.input("enter a message: ")
switch.set()
shell.printf("message sent")
def foo2(shell):
shell.printf("waiting for message...")
while shell.executing and not switch.is_set():
switch.wait(2) # by using shell.executing in the loop it will occasionally check
# if the program should quit because the window was closed
if shell.executing:
shell.printf("message recieved: ",x)
shell1 = ThreadShell(root,foo)
shell2 = ThreadShell(root,foo2)
first_time = True
while shell1.executing or shell2.executing or first_time:
first_time = False
root.mainloop()
root.destroy()
#!/usr/bin/env python
"""Show messages in two new console windows simultaneously."""
import sys
import platform
from subprocess import Popen
messages = 'This is Console1', 'This is Console2'
def randomFunction():
return "import sys; print(sys.argv[1]); input('Press Enter..')"
# define a command that starts new terminal
if platform.system() == "Windows":
new_window_command = "cmd.exe /c start".split()
else: #XXX this can be made more portable
new_window_command = "x-terminal-emulator -e".split()
# open new consoles, display messages
echo = [sys.executable, "-c",randomFunction()
]
processes = [Popen(new_window_command + echo + [msg]) for msg in messages]
# wait for the windows to be closed
for proc in processes:
proc.wait()
Find working solution for your problem, I haven't used thread, but can be done. And this solution is motivated from solution provided by "Miodrag Novakovic"
You have to change few paths as per your env. Below code is tested on windows
test_code.py -
import sys
import platform
from subprocess import Popen
messages = 'This is Console1', 'This is Console2'
def foo1():
print "In foo1"
i = input("Enter Something - ")
print i
input("Enter to exit")
def foo2():
print "In foo2"
i = input("Enter Something - ")
print i
input("Enter to exit")
def run_foo1():
print("foo1 >>> Something")
return "import sys; sys.path.append('path_to_your_program_folder'); from test_code import foo1; foo1()"
def run_foo2():
print("foo2 >>> Something")
return "import sys; sys.path.append('path_to_your_program_folder'); from test_code import foo2; foo2()"
# define a command that starts new terminal
if platform.system() == "Windows":
new_window_command = "cmd.exe /c start".split()
else: #XXX this can be made more portable
new_window_command = "x-terminal-emulator -e".split()
if __name__ == '__main__':
# open new consoles, display messages
echos = [[sys.executable, "-c",run_foo1()],
[sys.executable, "-c",run_foo2()]
]
processes = [Popen(new_window_command + echo) for echo in echos]
# wait for the windows to be closed
for proc in processes:
proc.wait()
My questions are interlaced within my code:
#!/usr/bin/python
import threading
import logging, logging.handlers
import hpclib
import json
import time
from datetime import datetime
from features import *
import sys
if len(sys.argv) != 3:
print "Please provide the correct inputs"
print "Usage: rest_test.py <controllerip> <counter>"
sys.exit()
controller = sys.argv[1]
counter = int(sys.argv[2])
class FuncThread(threading.Thread):
def __init__(self, target, *args):
self._target = target
self._args = args
threading.Thread.__init__(self)
def run(self):
self._target(*self._args)
def datapath_thread(ipaddress, testlogfile,count):
#initialize logging system
testlogger = logging.getLogger("testlogger")
testlogger.setLevel(logging.DEBUG)
file = open(testlogfile,'w')
file.close()
# This handler writes everything to a file.
h1 = logging.FileHandler(testlogfile)
f = logging.Formatter("%(levelname)s %(asctime)s %(funcName)s %(lineno)d %(message)s")
h1.setFormatter(f)
h1.setLevel(logging.DEBUG)
testlogger.addHandler(h1)
mylib = hpclib.hpclib(ipaddress)
success_count = 0
failure_count = 0
for i in range(count):
t1=datetime.now()
try:
(code, val) = datapaths.listDatapaths(mylib)
I want to pass this function datapaths.listDatapaths(mylib) as a argument from a thread below, something like (code,val)=functionname
if code == 200:
success_count +=1
else:
testlogger.debug("Return Code other than 200 received with code = %d, value = %s"%(code,val))
failure_count +=1
except:
failure_count += 1
testlogger.debug ("Unexpected error: %s"%sys.exc_info()[0])
continue
t2=datetime.now()
diff=t2-t1
testlogger.debug('RETURN code: %d. Time taken in sec = %s,Iteration = %d, Success = %d, Failure = %d'%(code,diff.seconds,i+1,success_count,failure_count))
time.sleep(1)
testlogger.removeHandler(h1)
# Passing ipadress of controller and log file name
t1 = FuncThread(datapath_thread, controller, "datapaths.log",counter)
Here I would like to pass function name as one of the argument,something like t1 = FuncThread(datapath_thread, controller, datapaths.listDatapaths(mylib),"datapaths.log",counter)
t1.start()
t1.join()
I have many functions to call like this,so want a easy way to call all the functions from one single function using many threads
Firstly, FuncThread is not very useful - FuncThread(func, *args) can be spelt Thread(target=lambda: func(*args)) or Thread(target=func, args=args).
You're pretty close - instead of passing in the result of calling the function, pass in the function itself
def datapath_thread(ipaddress, test_func, testlogfile, count):
# ...
for i in range(count):
# ...
try:
(code, val) = test_func(mylib)
#...
thread = Thread(target=datapath_thread, args=(
controller,
datapaths.listDatapaths,
"datapaths.log",
counter
))
I have to run a program on 200 files in a round robin.
Right now I have them running like this:
for combo in it.combinations(files, 2):
cmd = ["command", combo[0], combo[1]]
subprocess.Popen(cmd)
I would like to run only say 60 at a time as not to overwhelm the computer, the command is pretty processor intensive. What's the best way to pause the loop once 60 processes are running and then start again once one has finished so that there are always 60 processes running?
#!/usr/bin/env python
import itertools
import subprocess
from multiprocessing.dummy import Pool # use threads
def run(combo):
cmd = ["command", combo[0], combo[1]]
return combo, subprocess.call(cmd)
def main():
p = Pool(60) # 60 subprocesses at a time
for combo, rc in p.imap_unordered(run, itertools.combinations(files, 2)):
print("%s exited with %s" % (combo, rc))
p.close()
p.join()
if __name__ == "__main__":
main()
This answer demonstrates various techniques to limit number of concurrent subprocesses: it shows multiprocessing.Pool, concurrent.futures, threading + Queue -based solutions.
This might help:
import itertools as it
import time
import subprocess
files = range(5)
max_load = 3
sleep_interval = 0.5
pid_list = []
for combo in it.combinations(files, 2):
# Random command that takes time
cmd = ['sleep', str(combo[0]+combo[1])]
# Launch and record this command
print "Launching: ", cmd
pid = subprocess.Popen(cmd)
pid_list.append(pid)
# Deal with condtion of exceeding maximum load
while len(filter(lambda x: x.poll() is None, pid_list)) >= max_load:
time.sleep(sleep_interval)
You could do something really simple like:
from time import sleep
count = 0
for combo in it.combinations(files, 2):
while count < 60:
cmd = ["command", combo[0], combo[1]]
subprocess.Popen(cmd)
count = count + 1
if subprocess_is_done:
count = count - 1
sleep(5)
Obviously you'd need to figure out how to get subprocess_is_done from your command.
This works for trivial cases as far as I can tell, but have no clue what you're trying to run...
You want something like this:
import socket
import threading
import Queue
import subprocess
class IPThread(threading.Thread):
def __init__(self, queue, num):
super(IPThread, self).__init__()
self.queue = queue
self.num = num
def run(self):
while True:
try:
args = self.queue.get_nowait()
cmd = ["echo"] + [str(i) for i in args]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
print out
except Queue.Empty:
# Nothing left in the Queue -- we are done
print "Queue %d done" % self.num
break
except Exception as err:
# Handle exception
print err
self.queue.task_done()
def create_threads(q, size):
for i in range(size):
thread = IPThread(q, i)
thread.setDaemon(True)
thread.start()
q.join()
def fill_queue(q):
# Call q.put(args) in a loop to populate Queue with arguments
from itertools import permutations
x = list(range(20))
for arg1, arg2 in permutations(x, 2):
q.put([arg1, arg2])
print q.qsize()
def main():
q = Queue.Queue()
fill_queue(q)
create_threads(q, 60)
print "Done"
if __name__ == '__main__':
main()
Create a queue of things to work on. Specialize your Thread-derived class. Spin up your threads. Wait for them to be done.
You can tell that the tasks are running concurrently because their output interferes with each other. It's a feature!
I'm trying to get the clipboard content using a Python script on my Mac Lion.
I'm searching for an event or something similar, because if I use a loop, my application spends all its time watching the clipboard.
Any ideas?
Have you thought about using an endless loop and "sleeping" between tries?
I used pyperclip for a simple PoC and it worked like a charm, and Windows and Linux.
import time
import sys
import os
import pyperclip
recent_value = ""
while True:
tmp_value = pyperclip.paste()
if tmp_value != recent_value:
recent_value = tmp_value
print("Value changed: %s" % str(recent_value)[:20])
time.sleep(0.1)
Instead of the print, do whatever you want.
Here is a complete multithreading example.
import time
import threading
import pyperclip
def is_url_but_not_bitly(url):
if url.startswith("http://") and not "bit.ly" in url:
return True
return False
def print_to_stdout(clipboard_content):
print ("Found url: %s" % str(clipboard_content))
class ClipboardWatcher(threading.Thread):
def __init__(self, predicate, callback, pause=5.):
super(ClipboardWatcher, self).__init__()
self._predicate = predicate
self._callback = callback
self._pause = pause
self._stopping = False
def run(self):
recent_value = ""
while not self._stopping:
tmp_value = pyperclip.paste()
if tmp_value != recent_value:
recent_value = tmp_value
if self._predicate(recent_value):
self._callback(recent_value)
time.sleep(self._pause)
def stop(self):
self._stopping = True
def main():
watcher = ClipboardWatcher(is_url_but_not_bitly,
print_to_stdout,
5.)
watcher.start()
while True:
try:
print("Waiting for changed clipboard...")
time.sleep(10)
except KeyboardInterrupt:
watcher.stop()
break
if __name__ == "__main__":
main()
I create a subclass of threading.Thread, override the methods run and __init__ and create an instance of this class. By calling watcher.start() (not run()!), you start the thread.
To safely stop the thread, I wait for <Ctrl>-C (keyboard interrupt) and tell the thread to stop itself.
In the initialization of the class, you also have a parameter pause to control how long to wait between tries.
Use the class ClipboardWatcher like in my example, replace the callback with what you do, e.g., lambda x: bitly(x, username, password).
Looking at pyperclip the meat of it on Macosx is :
import os
def macSetClipboard(text):
outf = os.popen('pbcopy', 'w')
outf.write(text)
outf.close()
def macGetClipboard():
outf = os.popen('pbpaste', 'r')
content = outf.read()
outf.close()
return content
These work for me how do you get on?
I don't quite follow your comment on being in a loop.
EDIT Added 'orrid polling example that shows how changeCount() bumps up on each copy to the pasteboard. It's still not what the OP wants as there seems no event or notification for modifications to the NSPasteboard.
from LaunchServices import *
from AppKit import *
import os
from threading import Timer
def poll_clipboard():
pasteboard = NSPasteboard.generalPasteboard()
print pasteboard.changeCount()
def main():
while True:
t = Timer(1, poll_clipboard)
t.start()
t.join()
if __name__ == "__main__":
main()
simple!
import os
def macSetClipboard(text):
outf = os.popen('pbcopy', 'w')
outf.write(text)
outf.close()
def macGetClipboard():
outf = os.popen('pbpaste', 'r')
content = outf.read()
outf.close()
return content
current_clipboard = macGetClipboard()
while True:
clipboard = macGetClipboard()
if clipboard != current_clipboard:
print(clipboard)
macSetClipboard("my new string")
print(macGetClipboard())
break
I originaly posted my answer on a duplicate Run a python code when copying text with specific keyword
Here the answer I came up with.
import clipboard
import asyncio
# Exemple function.
async def your_function():
print("Running...")
async def wait4update(value):
while True:
if clipboard.paste() != value : # If the clipboard changed.
return
async def main():
value = clipboard.paste() # Set the default value.
while True :
update = asyncio.create_task(wait4update(value))
await update
value = clipboard.paste() # Change the value.
asyncio.create_task(your_function()) #Start your function.
asyncio.run(main())