Output sys.stdout and sys.stderr to a file - python

I have a module that redirects console outputs to the Tkinter window, which I found here on StackOverflow. It works perfectly, however, I realized I also need to save the console output to a file. But everything I tried either doesn't work, breaks the module, or nothing happens.
EDIT: I need to modify the following module so it also SAVES THE CONSOLE OUTPUT to the .txt document. BUT preserving it's current functionality
Here is the code for the module:
# Code derived from Bryan Olson's source posted in this related Usenet discussion:
# https://groups.google.com/d/msg/comp.lang.python/HWPhLhXKUos/TpFeWxEE9nsJ
# https://groups.google.com/d/msg/comp.lang.python/HWPhLhXKUos/eEHYAl4dH9YJ
#
# See the comments and doc string below.
#
# Here's a module to show stderr output from console-less Python
# apps, and stay out of the way otherwise. I plan to make a ASPN
# recipe of it, but I thought I'd run it by this group first.
#
# To use it, import the module. That's it. Upon import it will
# assign sys.stderr.
#
# In the normal case, your code is perfect so nothing ever gets
# written to stderr, and the module won't do much of anything.
# Upon the first write to stderr, if any, the module will launch a
# new process, and that process will show the stderr output in a
# window. The window will live until dismissed; I hate, hate, hate
# those vanishing-consoles-with-critical-information.
#
# The code shows some arguably-cool tricks. To fit everthing in
# one file, the module runs the Python interpreter on itself; it
# uses the "if __name__ == '__main__'" idiom to behave radically
# differently upon import versus direct execution. It uses tkinter
# for the window, but that's in a new process; it does not import
# tkinter into your application.
#
# To try it out, save it to a file -- I call it "errorwindow.py" -
# - and import it into some subsequently-incorrect code. For
# example:
#
# import errorwindow
#
# a = 3 + 1 + nonesuchdefined
#
# should cause a window to appear, showing the traceback of a
# Python NameError.
#
# --
# --Bryan
# ----------------------------------------------------------------
#
# martineau - Modified to use subprocess.Popen instead of the os.popen
# which has been deprecated since Py 2.6. Changed so it
# redirects both stdout and stderr. Added numerous
# comments, and also inserted double quotes around paths
# in case they have embedded space characters in them, as
# they did on my Windows system.
#
# Recently updated it to work in both Python 2 and Python 3.
"""
Import this module into graphical Python apps to provide a
sys.stderr. No functions to call, just import it. It uses
only facilities in the Python standard distribution.
If nothing is ever written to stderr, then the module just
sits there and stays out of your face. Upon write to stderr,
it launches a new process, piping it error stream. The new
process throws up a window showing the error messages.
"""
import subprocess
import sys
try:
import thread
except ModuleNotFoundError: # Python 3
import _thread as thread
import os
EXC_INFO_FILENAME = 'exc_info.txt'
if __name__ == '__main__': # When spawned as separate process.
# create window in which to display output
# then copy stdin to the window until EOF
# will happen when output is sent to each OutputPipe created
try:
from Tkinter import BOTH, END, Frame, Text, TOP, YES
import tkFont
import Queue
except ModuleNotFoundError: # Python 3
from tkinter import BOTH, END, Frame, Text, TOP, YES
import tkinter.font as tkFont
import queue as Queue
Q_EMPTY = Queue.Empty # An exception class.
queue = Queue.Queue(1000) # FIFO
def read_stdin(app, bufsize=4096):
fd = sys.stdin.fileno() # File descriptor for os.read() calls.
read = os.read
put = queue.put
while True:
put(read(fd, bufsize))
class Application(Frame):
def __init__(self, master=None, font_size=8, text_color='#0000AA', rows=25, cols=100):
Frame.__init__(self, master)
# Create title based on the arguments passed to the spawned script:
# argv[0]: name of this script (ignored)
# argv[1]: name of script that imported this module
# argv[2]: name of redirected stream (optional)
if len(sys.argv) < 2:
title = "Output stream from unknown source"
elif len(sys.argv) < 3:
title = "Output stream from %s" % (sys.argv[1],)
else: # Assume it's a least 3.
title = "Output stream '%s' from %s" % (sys.argv[2], sys.argv[1])
self.master.title(title)
self.pack(fill=BOTH, expand=YES)
font = tkFont.Font(family='Courier', size=font_size)
width = font.measure(' ' * (cols+1))
height = font.metrics('linespace') * (rows+1)
self.configure(width=width, height=height)
self.pack_propagate(0) # Force frame to be configured size.
self.logwidget = Text(self, font=font)
self.logwidget.pack(side=TOP, fill=BOTH, expand=YES)
# Disallow key entry, but allow text copying with <Control-c>
self.logwidget.bind('<Key>', lambda x: 'break')
self.logwidget.bind('<Control-c>', lambda x: None)
self.logwidget.configure(foreground=text_color)
self.logwidget.insert(END, '==== Start of Output Stream ====\n\n')
self.logwidget.see(END)
self.after(200, self.start_thread) # Start queue polling thread.
def start_thread(self):
thread.start_new_thread(read_stdin, (self,))
self.after(200, self.check_q)
def check_q(self):
log = self.logwidget
log_insert = log.insert
log_see = log.see
queue_get_nowait = queue.get_nowait
go = True
while go:
try:
data = queue_get_nowait().decode() # Must decode for Python 3.
if not data:
data = '[EOF]'
go = False
log_insert(END, data)
log_see(END)
except Q_EMPTY:
self.after(200, self.check_q)
go = False
app = Application()
app.mainloop()
else: # when module is first imported
import traceback
class OutputPipe(object):
def __init__(self, name=''):
self.lock = thread.allocate_lock()
self.name = name
def flush(self): # no-op.
pass
def __getattr__(self, attr):
if attr == 'pipe': # Attribute doesn't exist, so create it.
# Launch this module as a separate process to display any output
# it receives.
# Note: It's important to put double quotes around everything in
# case any have embedded space characters.
command = '"%s" "%s" "%s" "%s"' % (sys.executable, # executable
__file__, # argv[0]
os.path.basename(sys.argv[0]), # argv[1]
self.name) # argv[2]
#
# Typical command and arg values on receiving end:
# C:\Python3\python[w].exe # executable
# C:\vols\Files\PythonLib\Stack Overflow\errorwindow3k.py # argv[0]
# errorwindow3k_test.py # argv[1]
# stderr # argv[2]
# Execute this script directly as __main__ with a stdin PIPE for sending
# output to it.
try:
# Had to also make stdout and stderr PIPEs too, to work with pythonw.exe
self.pipe = subprocess.Popen(command, bufsize=0,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).stdin
except Exception:
# Output exception info to a file since this module isn't working.
exc_type, exc_value, exc_traceback = sys.exc_info()
msg = ('%r exception in %s\n' %
(exc_type.__name__, os.path.basename(__file__)))
with open(EXC_INFO_FILENAME, 'wt') as info:
info.write('fatal error occurred spawning output process')
info.write('exeception info:' + msg)
traceback.print_exc(file=info)
sys.exit('fatal error occurred')
return super(OutputPipe, self).__getattribute__(attr)
def write(self, data):
with self.lock:
data = data.encode() # Must encode for Python 3.
self.pipe.write(data) # First reference to pipe attr will cause an
# OutputPipe process for the stream to be created.
# Clean-up any left-over debugging file.
try:
os.remove(EXC_INFO_FILENAME) # Delete previous file, if any.
except Exception:
pass
# Redirect standard output streams in the process that imported this module.
sys.stderr = OutputPipe('stderr')
sys.stdout = OutputPipe('stdout')
I tried inserting this code after, before, and in between the module, but it didn't work or broke the module:
import sys path = 'output.txt' sys.stdout = open(path, 'w')
I also tried to do something like this, but it didn't work either.
f = open("output.txt", "w")
f.write(sys.stdout) # or "f.write(OutputPipe('stdout')" or " 'f.write(data)' between - 'def write(self, data):' "
f.close()

try:
sys.stdout = open('file.txt', 'w')
# some working code
finally:
# close file.txt
sys.stdout.close()
sys.stdout = sys.__stdout__
Update. You can write your custom logging class to suit your needs
import sys
class Logger:
def __init__(self, filename):
self.console = sys.stdout
self.file = open(filename, 'w')
def write(self, message):
self.console.write(message)
self.file.write(message)
def flush(self):
self.console.flush()
self.file.flush()
sys.stdout = Logger('file.txt')

I managed to solve my issue by adding a few lines to the code. I'll share here the full modified module code in the unlikely case that someone stumbles upon this same issue.
# Code derived from Bryan Olson's source posted in this related Usenet discussion:
# https://groups.google.com/d/msg/comp.lang.python/HWPhLhXKUos/TpFeWxEE9nsJ
# https://groups.google.com/d/msg/comp.lang.python/HWPhLhXKUos/eEHYAl4dH9YJ
#
# See the comments and doc string below.
#
# Here's a module to show stderr output from console-less Python
# apps, and stay out of the way otherwise. I plan to make a ASPN
# recipe of it, but I thought I'd run it by this group first.
#
# To use it, import the module. That's it. Upon import it will
# assign sys.stderr.
#
# In the normal case, your code is perfect so nothing ever gets
# written to stderr, and the module won't do much of anything.
# Upon the first write to stderr, if any, the module will launch a
# new process, and that process will show the stderr output in a
# window. The window will live until dismissed; I hate, hate, hate
# those vanishing-consoles-with-critical-information.
#
# The code shows some arguably-cool tricks. To fit everthing in
# one file, the module runs the Python interpreter on itself; it
# uses the "if __name__ == '__main__'" idiom to behave radically
# differently upon import versus direct execution. It uses tkinter
# for the window, but that's in a new process; it does not import
# tkinter into your application.
#
# To try it out, save it to a file -- I call it "errorwindow.py" -
# - and import it into some subsequently-incorrect code. For
# example:
#
# import errorwindow
#
# a = 3 + 1 + nonesuchdefined
#
# should cause a window to appear, showing the traceback of a
# Python NameError.
#
# --
# --Bryan
# ----------------------------------------------------------------
#
# martineau - Modified to use subprocess.Popen instead of the os.popen
# which has been deprecated since Py 2.6. Changed so it
# redirects both stdout and stderr. Added numerous
# comments, and also inserted double quotes around paths
# in case they have embedded space characters in them, as
# they did on my Windows system.
#
# Recently updated it to work in both Python 2 and Python 3.
"""
Import this module into graphical Python apps to provide a
sys.stderr. No functions to call, just import it. It uses
only facilities in the Python standard distribution.
If nothing is ever written to stderr, then the module just
sits there and stays out of your face. Upon write to stderr,
it launches a new process, piping it error stream. The new
process throws up a window showing the error messages.
"""
import subprocess
import sys
try:
import thread
except ModuleNotFoundError: # Python 3
import _thread as thread
import os
EXC_INFO_FILENAME = 'exc_info.txt'
if __name__ == '__main__': # When spawned as separate process.
# create window in which to display output
# then copy stdin to the window until EOF
# will happen when output is sent to each OutputPipe created
try:
from Tkinter import BOTH, END, Frame, Text, TOP, YES
import tkFont
import Queue
except ModuleNotFoundError: # Python 3
from tkinter import BOTH, END, Frame, Text, TOP, YES
import tkinter.font as tkFont
import queue as Queue
Q_EMPTY = Queue.Empty # An exception class.
queue = Queue.Queue(1000) # FIFO
def read_stdin(app, bufsize=4096):
fd = sys.stdin.fileno() # File descriptor for os.read() calls.
read = os.read
put = queue.put
while True:
put(read(fd, bufsize))
class Application(Frame):
def __init__(self, master=None, font_size=8, text_color='#0000AA', rows=25, cols=100):
Frame.__init__(self, master)
# Create title based on the arguments passed to the spawned script:
# argv[0]: name of this script (ignored)
# argv[1]: name of script that imported this module
# argv[2]: name of redirected stream (optional)
if len(sys.argv) < 2:
title = "Output stream from unknown source"
elif len(sys.argv) < 3:
title = "Output stream from %s" % (sys.argv[1],)
else: # Assume it's a least 3.
title = "Output stream '%s' from %s" % (sys.argv[2], sys.argv[1])
self.master.title(title)
self.pack(fill=BOTH, expand=YES)
font = tkFont.Font(family='Courier', size=font_size)
width = font.measure(' ' * (cols+1))
height = font.metrics('linespace') * (rows+1)
self.configure(width=width, height=height)
self.pack_propagate(0) # Force frame to be configured size.
self.logwidget = Text(self, font=font)
self.logwidget.pack(side=TOP, fill=BOTH, expand=YES)
# Disallow key entry, but allow text copying with <Control-c>
self.logwidget.bind('<Key>', lambda x: 'break')
self.logwidget.bind('<Control-c>', lambda x: None)
self.logwidget.configure(foreground=text_color)
self.logwidget.insert(END, '==== Start of Output Stream ====\n\n')
# Writes the console output to file!!! :) MOD
with open("Output.txt", "a") as f:
f.write('==== Start of Output Stream ====\n\n')
self.logwidget.see(END)
self.after(200, self.start_thread) # Start queue polling thread.
def start_thread(self):
thread.start_new_thread(read_stdin, (self,))
self.after(200, self.check_q)
def check_q(self):
log = self.logwidget
log_insert = log.insert
log_see = log.see
queue_get_nowait = queue.get_nowait
go = True
while go:
try:
data = queue_get_nowait().decode() # Must decode for Python 3.
with open("Output.txt", "a") as f: # Writes the console output to file!!! :) MOD
f.write(str(data))
if not data:
data = '[EOF]'
go = False
with open("Output.txt", "a") as f: # Writes the '[EOF]' to file!!! :) MOD
f.write(str('\n' + '============ [EOF] =============' + '\n' + '\n' + '\n'))
log_insert(END, data)
log_see(END)
except Q_EMPTY:
self.after(200, self.check_q)
go = False
app = Application()
app.mainloop()
else: # when module is first imported
import traceback
class OutputPipe(object):
def __init__(self, name=''):
self.lock = thread.allocate_lock()
self.name = name
def flush(self): # no-op.
pass
def __getattr__(self, attr):
if attr == 'pipe': # Attribute doesn't exist, so create it.
# Launch this module as a separate process to display any output
# it receives.
# Note: It's important to put double quotes around everything in
# case any have embedded space characters.
command = '"%s" "%s" "%s" "%s"' % (sys.executable, # executable
__file__, # argv[0]
os.path.basename(sys.argv[0]), # argv[1]
self.name) # argv[2]
#
# Typical command and arg values on receiving end:
# C:\Python3\python[w].exe # executable
# C:\vols\Files\PythonLib\Stack Overflow\errorwindow3k.py # argv[0]
# errorwindow3k_test.py # argv[1]
# stderr # argv[2]
# Execute this script directly as __main__ with a stdin PIPE for sending
# output to it.
try:
# Had to also make stdout and stderr PIPEs too, to work with pythonw.exe
self.pipe = subprocess.Popen(command, bufsize=0,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).stdin
except Exception:
# Output exception info to a file since this module isn't working.
exc_type, exc_value, exc_traceback = sys.exc_info()
msg = ('%r exception in %s\n' %
(exc_type.__name__, os.path.basename(__file__)))
with open(EXC_INFO_FILENAME, 'wt') as info:
info.write('fatal error occurred spawning output process')
info.write('exeception info:' + msg)
traceback.print_exc(file=info)
sys.exit('fatal error occurred')
return super(OutputPipe, self).__getattribute__(attr)
def write(self, data):
with self.lock:
data = data.encode() # Must encode for Python 3.
self.pipe.write(data) # First reference to pipe attr will cause an
# OutputPipe process for the stream to be created.
# Clean-up any left-over debugging file.
try:
os.remove(EXC_INFO_FILENAME) # Delete previous file, if any.
except Exception:
pass
# Redirect standard output streams in the process that imported this module.
sys.stderr = OutputPipe('stderr')
sys.stdout = OutputPipe('stdout')

Related

How to make tkinter text box a stdin input receiver?

I made two tkinter text boxes one of which takes your python script as input and the other one shows results of the execution of your script but when I used input() command I got an error. Below given is the class for stdout redirector and also the execute function which executes after reading the script, which works fine. I have not included Text, tkinter, etc because I use all the general methods that work with the code like Text.get(), Text.mark_set(), Text.replace(), etc and also some of the functions are not included here. Other than the script and the output boxes I also tried to embed whole of the console in a textbox with InteractiveConsole but the problem was same in the case of receiving input or stdin but in both the cases stdout and stderr works fine.
from code import InteractiveConsole, InteractiveInterpreter
class StdoutRedirector(object):
def __init__(self, text_widget):
self.text_space = text_widget
def write(self, string):
self.text_space.insert('end', string)
self.text_space.see('end')
##class StdinRedirector(object):
## def __init__(self, text_widget):
## self.text_space = text_widget
##
## def readline(self) -> str:
## t = self.text_space.get(INSERT, f"{int(text.index(INSERT).split('.')[0])}.{int(text.index(INSERT).split('.')[1])}")
## return t
def execute(event=None):
save()
code = text.get('1.0',END+'-1c')
stdin = sys.stdin
stdout = sys.stdout
stderr = sys.stderr
output.delete('1.0',END)
## def a():
## sys.stdin = StdinRedirector(output)
## output.bind('<Return>', lambda: a)
sys.stdout = StdoutRedirector(output)
sys.stderr = StdoutRedirector(output)
interp = InteractiveInterpreter()
interp.runcode(code)
sys.stdout = stdout
sys.stderr = stderr
## sys.stdin = stdin
After which I tried Redirecting stdin, which obviously didn't work, and instead the application hung and the window stopped responding even after trying again and again.
Please help me with this... I don't know if its impossible but PyCharm and others have I/O Streams inside them so maybe the console or the execution window CAN be wholly embedded in a text box.
Ok so after researching on the web, in docs, and inside the code of the queue, idlelib and subprocess modules, I figured out the simplest way to make tkinter Textbox interact with python console as stdin, stdout, and stderr receiver. Here's the code:
import tkinter as tk
import subprocess
import queue
import os
from threading import Thread
class Console(tk.Frame):
def __init__(self, parent=None, **kwargs):
tk.Frame.__init__(self, parent, **kwargs)
self.parent = parent
# create widgets
self.ttytext = tk.Text(self, wrap=tk.WORD)
self.ttytext.pack(fill=tk.BOTH, expand=True)
self.ttytext.linenumbers.pack_forget()
self.p = subprocess.Popen(["jupyter", "qtconsole"], stdout=subprocess.PIPE, stdin=subprocess.PIPE,
stderr=subprocess.PIPE, creationflags=subprocess.CREATE_NO_WINDOW)
# make queues for keeping stdout and stderr whilst it is transferred between threads
self.outQueue = queue.Queue()
self.errQueue = queue.Queue()
# keep track of where any line that is submitted starts
self.line_start = 0
# a daemon to keep track of the threads so they can stop running
self.alive = True
# start the functions that get stdout and stderr in separate threads
Thread(target=self.readfromproccessout).start()
Thread(target=self.readfromproccesserr).start()
# start the write loop in the main thread
self.writeloop()
# key bindings for events
self.ttytext.bind("<Return>", self.enter)
self.ttytext.bind('<BackSpace>', self.on_bkspace)
self.ttytext.bind('<Delete>', self.on_delete)
self.ttytext.bind('<<Copy>>', self.on_copy)
self.ttytext.bind('<<Paste>>', self.on_paste)
self.ttytext.bind('<Control-c>', self.on_copy)
self.ttytext.bind('<Control-v>', self.on_paste)
def destroy(self):
"""This is the function that is automatically called when the widget is destroyed."""
self.alive = False
# write exit() to the console in order to stop it running
self.p.stdin.write("exit()\n".encode())
self.p.stdin.flush()
# call the destroy methods to properly destroy widgets
self.ttytext.destroy()
tk.Frame.destroy(self)
def enter(self, event):
"""The <Return> key press handler"""
cur_ind = str(self.ttytext.index(tk.INSERT))
if int(cur_ind.split('.')[0]) < int(self.ttytext.search(': ', tk.END, backwards=True).split('.')[0]):
try:
selected = self.ttytext.get('sel.first', 'sel.last')
if len(selected) > 0:
self.ttytext.insert(tk.END, selected)
self.ttytext.mark_set(tk.INSERT, tk.END)
self.ttytext.see(tk.INSERT)
return 'break'
except:
selected = self.ttytext.get(
self.ttytext.search(': ', tk.INSERT, backwards=True), tk.INSERT)
self.ttytext.insert(tk.END, selected.strip(': '))
self.ttytext.mark_set(tk.INSERT, tk.END)
self.ttytext.see(tk.INSERT)
return 'break'
string = self.ttytext.get(1.0, tk.END)[self.line_start:]
self.line_start += len(string)
self.p.stdin.write(string.encode())
self.p.stdin.flush()
def on_bkspace(self, event):
pass
def on_delete(self, event):
pass
def on_key(self, event):
"""The typing control (<KeyRelease>) handler"""
cur_ind = str(self.ttytext.index(tk.INSERT))
try:
if int(cur_ind.split('.')[0]) < int(self.ttytext.search(r'In [0-9]?', tk.END, backwards=True).split('.')[0]):
return 'break'
except:
return
def on_copy(self, event):
"""<Copy> event handler"""
self.ttytext.clipboard_append(self.ttytext.get('sel.first', 'sel.last'))
# I created this function because I was going to make a custom textbox
def on_paste(self, event):
"""<Paste> event handler"""
self.ttytext.insert(tk.INSERT, self.ttytext.clipboard_get())
# I created this function because I was going to make a custom textbox
def readfromproccessout(self):
"""To be executed in a separate thread to make read non-blocking"""
while self.alive:
data = self.p.stdout.raw.read(1024).decode()
self.outQueue.put(data)
def readfromproccesserr(self):
"""To be executed in a separate thread to make read non-blocking"""
while self.alive:
data = self.p.stderr.raw.read(1024).decode()
self.errQueue.put(data)
def writeloop(self):
"""Used to write data from stdout and stderr to the Text widget"""
# if there is anything to write from stdout or stderr, then write it
if not self.errQueue.empty():
self.write(self.errQueue.get())
if not self.outQueue.empty():
self.write(self.outQueue.get())
# run this method again after 10ms
if self.alive:
self.after(10, self.writeloop)
def write(self, string):
self.ttytext.insert(tk.END, string)
self.ttytext.see(tk.END)
self.line_start += len(string)
self.ttytext.inst_trigger()
if __name__ == '__main__':
root = tk.Tk()
main_window = Console(root)
main_window.pack(fill=tk.BOTH, expand=True)
main_window.ttytext.focus_force()
root.mainloop()
The code above uses jupyter qtconsole (because it is very handy), otherwise simple python shell can also be used using the InteractiveShell() in code module.
I have not completely made functions for Enter key, Up and Down arrow keys. These can be made by the user as per their choice.
This can also be found in Oli's answer here and it is customizable.

Redirect all stdout/stderr globally to logger

Background
I have a very large python application that launches command-line utilities to get pieces of data it needs. I currently just redirect the python launcher script to a log file, which gives me all of the print() output, plus the output of the command-line utilities, i.e.:
python -m launcher.py &> /root/out.log
Problem
I've since implemented a proper logger via logging, which lets me format the logging statements more precisely, lets me limit log file size, etc. I've swapped out most of my print()statements with calls to my logger. However, I have a problem: none of the output from the command-line applications is appearing in my log. It instead gets dumped to the console. Also, the programs aren't all launched the same way: some are launched via popen(), some by exec(), some by os.system(), etc.
Question
Is there a way to globally redirect all stdout/stderr text to my logging function, without having to re-write/modify the code that launches these command-line tools? I tried setting setting the following which I found in another question:
sys.stderr.write = lambda s: logger.error(s)
However it fails with "sys.stderr.write is read-only".
While this is not a full answer, it may show you a redirect to adapt to your particular case. This is how I did it a while back. Although I cannot remember why I did it this way, or what the limitation was I was trying to circumvent, the following is redirecting stdout and stderr to a class for print() statements. The class subsequently writes to screen and to file:
import os
import sys
import datetime
class DebugLogger():
def __init__(self, filename):
timestamp = datetime.datetime.strftime(datetime.datetime.utcnow(),
'%Y-%m-%d-%H-%M-%S-%f')
#build up full path to filename
logfile = os.path.join(os.path.dirname(sys.executable),
filename + timestamp)
self.terminal = sys.stdout
self.log = open(logfile, 'a')
def write(self, message):
timestamp = datetime.datetime.strftime(datetime.datetime.utcnow(),
' %Y-%m-%d-%H:%M:%S.%f')
#write to screen
self.terminal.write(message)
#write to file
self.log.write(timestamp + ' - ' + message)
self.flush()
def flush(self):
self.terminal.flush()
self.log.flush()
os.fsync(self.log.fileno())
def close(self):
self.log.close()
def main(debug = False):
if debug:
filename = 'blabla'
sys.stdout = DebugLogger(filename)
sys.stderr = sys.stdout
print('test')
if __name__ == '__main__':
main(debug = True)
import sys
import io
class MyStream(io.IOBase):
def write(self, s):
logger.error(s)
sys.stderr = MyStream()
print('This is an error', stream=sys.stderr)
This make all call to sys.stderr go to the logger.
The original one is always in sys.__stderr__

Python StringIO is not correctly capturing data from stderr

I have written some unittests that analyze data that is logged with the standard python logging function. Using some of the ideas that I found here: Capture stdout from a script in Python about how to capture data from stderr, I have come up with the following script, which I have simplified down to the bare minimum to illustrate a problem that I have encountered. (the loop below is simulates the fact that this function might be called from various unittests)
import logging, sys
from StringIO import StringIO
def get_stderr():
saved_stderr = sys.stderr
stderr_string_io = StringIO()
sys.stderr = stderr_string_io
try:
logging.error("Foobar!!!")
finally:
# set the stdout and stderr back to their original values
sys.stderr = saved_stderr
err_output = stderr_string_io.getvalue()
return err_output
for x in [1, 2]:
err_output = get_stderr()
print "Run %d: %s" % (x, err_output)
If you run the script it will give the following output, in which the logging output from the second loop iteration is totally lost:
Run 1: ERROR:root:Foobar!!!
Run 2:
Process finished with exit code 0
While I would expect it to give the following output:
Run 1: ERROR:root:Foobar!!!
Run 2: ERROR:root:Foobar!!!
Process finished with exit code 0
Note: that executing stderr_string_io.close() at the end of the function does not work, as the script then throws an ValueError the next time the function is executed.
Why does this code not behave as expected, and what is the solution to correct this problem?
When you call
logging.error
it runs
def error(msg, *args, **kwargs):
if len(root.handlers) == 0:
basicConfig()
root.error(msg, *args, **kwargs)
Since there are no root handlers at the start, it runs basicConfig with no arguments, which does:
def basicConfig():
_acquireLock()
try:
if len(root.handlers) == 0:
h = StreamHandler(None)
handlers = [h]
dfs = None
style = '%'
fs = kwargs.get("format", _STYLES[style][1])
fmt = Formatter(fs, dfs, style)
for h in handlers:
if h.formatter is None:
h.setFormatter(fmt)
root.addHandler(h)
finally:
_releaseLock()
I've removed code that can't run when you have no arguments.
So this has set handlers = [StreamHandler(None)]:
class StreamHandler(Handler):
def __init__(self, stream=None):
Handler.__init__(self)
if stream is None:
stream = sys.stderr
self.stream = stream
which means that you have the top level logger permanantly attached to whatever was stdout at the time you called it.
This causes your problem, because you throw away that output. This means that the output will go to a dead StringIO object, and be lost.
One way to deal with this is to go through handlers when updating stderr and also replace anything that refers to stderr:
import logging, sys
from StringIO import StringIO
def get_stderr():
saved_stderr = sys.stderr
stderr_string_io = StringIO()
for handler in logging.root.handlers:
if handler.stream is sys.stderr:
handler.stream = stderr_string_io
sys.stderr = stderr_string_io
try:
logging.error("Foobar!!!")
finally:
# set the stdout and stderr back to their original values
for handler in logging.root.handlers:
if handler.stream is sys.stderr:
handler.stream = saved_stderr
sys.stderr = saved_stderr
err_output = stderr_string_io.getvalue()
return err_output
for x in [1, 2]:
err_output = get_stderr()
print "Run %d: %s" % (x, err_output)
I don't know how well this will work. It also won't catch any loggers that aren't root loggers. Personally the idea of capturing sys.stdout by value is absurd, and this seems like an inevitable result.

Writing to easygui textbox as function is running?

I am currently running a script using easygui to receive the user input. The old script that ran in a command line would just print anything the user needed to know in the command line, but I've changed it to output notifications in new easygui boxes when there are required inputs.
Want I want to do is have the progress, each action inside the functions that are running, print into a text box as they complete. In command line I could just use print "text" but I can't get it to happen realtime in easygui. Currently I'm appending a list so I have a textbox that displays the results of the function once everything is complete, but I'd like for the large textbox window to pop up and print the line out whenever a process of note completes. Is this doable?
Here is how I'm appending the list:
result_list = []
record_str = "\n Polling has completed for 502."
result_list.append(record_str)
eg.textbox("Polling Status", "Daily polling completion status:", result_list)
I don't think there's any simple way of getting EasyGUI's textbox function to do what you want short of modifying the module. Since it's a function not a class, you can't even derive a subclass from it in order to easily reuse its code.
However it's completely feasible to create a separate Tkinter window that just displays lines of text as they are sent to it using an enhanced version of some code I found once in a thread on the comp.lang.python newsgroup.
The original code was designed to catch and display only stderr output from a GUI application which normally doesn't have stderr output handle, so the module was named errorwindow. However I modified it to be able redirect both stderr and stdout to such windows in one easygui-based app I developed, but I never got around to renaming it or updating the comments in it to also mention stdout redirection. ;¬)
Anyway, the module works by defining and creating two instances of a file-like class named OutputPipe when it's imported and assigns them to the sys.stdout and sys.stderr I/O stream file objects which are usually None in a Python .pyw GUI applications (on Windows). When output is first sent to either one of these, the same module is launched as separate Python process with its stdin, stdout, and stderr I/O handles piped with the original process.
There's a lot going on, but if nothing else, with a little study of it might give you some ideas about how to get easygui's textbox to do what you want. Hope this helps.
Note: The code posted is for Python 2.x, there's a modified version that will work in both Python 2 and 3 in my answer to another question, if anyone is interested.
File errorwindow.py:
# Code derived from Bryan Olson's source posted in this related Usenet discussion:
# https://groups.google.com/d/msg/comp.lang.python/HWPhLhXKUos/TpFeWxEE9nsJ
# https://groups.google.com/d/msg/comp.lang.python/HWPhLhXKUos/eEHYAl4dH9YJ
#
# Here's a module to show stderr output from console-less Python
# apps, and stay out of the way otherwise. I plan to make a ASPN
# recipe of it, but I thought I'd run it by this group first.
#
# To use it, import the module. That's it. Upon import it will
# assign sys.stderr.
#
# In the normal case, your code is perfect so nothing ever gets
# written to stderr, and the module won't do much of anything.
# Upon the first write to stderr, if any, the module will launch a
# new process, and that process will show the stderr output in a
# window. The window will live until dismissed; I hate, hate, hate
# those vanishing-consoles-with-critical-information.
#
# The code shows some arguably-cool tricks. To fit everthing in
# one file, the module runs the Python interpreter on itself; it
# uses the "if __name__ == '__main__'" idiom to behave radically
# differently upon import versus direct execution. It uses TkInter
# for the window, but that's in a new process; it does not import
# TkInter into your application.
#
# To try it out, save it to a file -- I call it "errorwindow.py" -
# - and import it into some subsequently-incorrect code. For
# example:
#
# import errorwindow
#
# a = 3 + 1 + nonesuchdefined
#
# should cause a window to appear, showing the traceback of a
# Python NameError.
#
# --
# --Bryan
# ----------------------------------------------------------------
#
# martineau - Modified to use subprocess.Popen instead of the os.popen
# which has been deprecated since Py 2.6. Changed so it
# redirects both stdout and stderr. Added numerous
# comments, and also inserted double quotes around paths
# in case they have embedded space characters in them, as
# they did on my Windows system.
"""
Import this module into graphical Python apps to provide a
sys.stderr. No functions to call, just import it. It uses
only facilities in the Python standard distribution.
If nothing is ever written to stderr, then the module just
sits there and stays out of your face. Upon write to stderr,
it launches a new process, piping it error stream. The new
process throws up a window showing the error messages.
"""
import subprocess
import sys
import thread
import os
if __name__ == '__main__': # when spawned as separate process
# create window in which to display output
# then copy stdin to the window until EOF
# will happen when output is sent to each OutputPipe created
from Tkinter import BOTH, END, Frame, Text, TOP, YES
import tkFont
import Queue
queue = Queue.Queue(100)
def read_stdin(app, bufsize=4096):
fd = sys.stdin.fileno() # gets file descriptor
read = os.read
put = queue.put
while True:
put(read(fd, bufsize))
class Application(Frame):
def __init__(self, master=None, font_size=8, text_color='#0000AA', rows=25, cols=100):
Frame.__init__(self, master)
# argv[0]: name of this script (not used)
# argv[1]: name of script that imported this module
# argv[2]: name of redirected stream (optional)
if len(sys.argv) < 3:
title = "Output Stream from %s" % (sys.argv[1],)
else:
title = "Output Stream '%s' from %s" % (sys.argv[2], sys.argv[1])
self.master.title(title)
self.pack(fill=BOTH, expand=YES)
font = tkFont.Font(family='Courier', size=font_size)
width = font.measure(' '*(cols+1))
height = font.metrics('linespace')*(rows+1)
self.configure(width=width, height=height)
self.pack_propagate(0) # force frame to be configured size
self.logwidget = Text(self, font=font)
self.logwidget.pack(side=TOP, fill=BOTH, expand=YES)
# Disallow key entry, but allow copy with <Control-c>
self.logwidget.bind('<Key>', lambda x: 'break')
self.logwidget.bind('<Control-c>', lambda x: None)
self.logwidget.configure(foreground=text_color)
#self.logwidget.insert(END, '==== Start of Output Stream ====\n\n')
#self.logwidget.see(END)
self.after(200, self.start_thread, ())
def start_thread(self, _):
thread.start_new_thread(read_stdin, (self,))
self.after(200, self.check_q, ())
def check_q(self, _):
log = self.logwidget
log_insert = log.insert
log_see = log.see
queue_get_nowait = queue.get_nowait
go = True
while go:
try:
data = queue_get_nowait()
if not data:
data = '[EOF]'
go = False
log_insert(END, data)
log_see(END)
except Queue.Empty:
self.after(200, self.check_q, ())
go = False
app = Application()
app.mainloop()
else: # when module is first imported
import traceback
class OutputPipe(object):
def __init__(self, name=''):
self.lock = thread.allocate_lock()
self.name = name
def __getattr__(self, attr):
if attr == 'pipe': # pipe attribute hasn't been created yet
# launch this module as a separate process to display any output
# it receives.
# Note: It's important to put double quotes around everything in case
# they have embedded space characters.
command = '"%s" "%s" "%s" "%s"' % (sys.executable, # command
__file__, # argv[0]
os.path.basename(sys.argv[0]), # argv[1]
self.name) # argv[2]
# sample command and arg values on receiving end:
# E:\Program Files\Python\python[w].exe # command
# H:\PythonLib\TestScripts\PyRemindWrk\errorwindow.py # argv[0]
# errorwindow.py # argv[1]
# stderr # argv[2]
# execute this script as __main__ with a stdin PIPE for sending output to it
try:
# had to make stdout and stderr PIPEs too, to make it work with pythonw.exe
self.pipe = subprocess.Popen(command, bufsize=0,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).stdin
except Exception:
# output exception info to a file since this module isn't working
exc_type, exc_value, exc_traceback = sys.exc_info()
msg = ('%r exception in %s\n' %
(exc_type.__name__, os.path.basename(__file__)))
with open('exc_info.txt', 'wt') as info:
info.write('msg:' + msg)
traceback.print_exc(file=info)
sys.exit('fatal error occurred spawning output process')
return super(OutputPipe, self).__getattribute__(attr)
def write(self, data):
with self.lock:
self.pipe.write(data) # 1st reference to pipe attr will cause it to be created
# redirect standard output streams in the process importing the module
sys.stderr = OutputPipe('stderr')
sys.stdout = OutputPipe('stdout')

How to capture frames from Apple iSight using Python and PyObjC?

I am trying to capture a single frame from the Apple iSight camera built into a Macbook Pro using Python (version 2.7 or 2.6) and the PyObjC (version 2.2).
As a starting point, I used this old StackOverflow question. To verify that it makes sense, I cross-referenced against Apple's MyRecorder example that it seems to be based on. Unfortunately, my script does not work.
My big questions are:
Am I initializing the camera correctly?
Am I starting the event loop correctly?
Was there any other setup I was supposed to do?
In the example script pasted below, the intended operation is that after calling startImageCapture(), I should start printing "Got a frame..." messages from the CaptureDelegate. However, the camera's light never turns on and the delegate's callback is never executed.
Also, there are no failures during startImageCapture(), all functions claim to succeed, and it successfully finds the iSight device. Analyzing the session object in pdb shows that it has valid input and output objects, the output has a delegate assigned, the device is not in use by another processes, and the session is marked as running after startRunning() is called.
Here's the code:
#!/usr/bin/env python2.7
import sys
import os
import time
import objc
import QTKit
import AppKit
from Foundation import NSObject
from Foundation import NSTimer
from PyObjCTools import AppHelper
objc.setVerbose(True)
class CaptureDelegate(NSObject):
def captureOutput_didOutputVideoFrame_withSampleBuffer_fromConnection_(self, captureOutput,
videoFrame, sampleBuffer,
connection):
# This should get called for every captured frame
print "Got a frame: %s" % videoFrame
class QuitClass(NSObject):
def quitMainLoop_(self, aTimer):
# Just stop the main loop.
print "Quitting main loop."
AppHelper.stopEventLoop()
def startImageCapture():
error = None
# Create a QT Capture session
session = QTKit.QTCaptureSession.alloc().init()
# Find iSight device and open it
dev = QTKit.QTCaptureDevice.defaultInputDeviceWithMediaType_(QTKit.QTMediaTypeVideo)
print "Device: %s" % dev
if not dev.open_(error):
print "Couldn't open capture device."
return
# Create an input instance with the device we found and add to session
input = QTKit.QTCaptureDeviceInput.alloc().initWithDevice_(dev)
if not session.addInput_error_(input, error):
print "Couldn't add input device."
return
# Create an output instance with a delegate for callbacks and add to session
output = QTKit.QTCaptureDecompressedVideoOutput.alloc().init()
delegate = CaptureDelegate.alloc().init()
output.setDelegate_(delegate)
if not session.addOutput_error_(output, error):
print "Failed to add output delegate."
return
# Start the capture
print "Initiating capture..."
session.startRunning()
def main():
# Open camera and start capturing frames
startImageCapture()
# Setup a timer to quit in 10 seconds (hack for now)
quitInst = QuitClass.alloc().init()
NSTimer.scheduledTimerWithTimeInterval_target_selector_userInfo_repeats_(10.0,
quitInst,
'quitMainLoop:',
None,
False)
# Start Cocoa's main event loop
AppHelper.runConsoleEventLoop(installInterrupt=True)
print "After event loop"
if __name__ == "__main__":
main()
Thanks for any help you can provide!
OK, I spent a day diving through the depths of PyObjC and got it working.
For future record, the reason the code in the question did not work: variable scope and garbage collection. The session variable was deleted when it fell out of scope, which happened before the event processor ran. Something must be done to retain it so it is not freed before it has time to run.
Moving everything into a class and making session a class variable made the callbacks start working. Additionally, the code below demonstrates getting the frame's pixel data into bitmap format and saving it via Cocoa calls, and also how to copy it back into Python's world-view as a buffer or string.
The script below will capture a single frame
#!/usr/bin/env python2.7
#
# camera.py -- by Trevor Bentley (02/04/2011)
#
# This work is licensed under a Creative Commons Attribution 3.0 Unported License.
#
# Run from the command line on an Apple laptop running OS X 10.6, this script will
# take a single frame capture using the built-in iSight camera and save it to disk
# using three methods.
#
import sys
import os
import time
import objc
import QTKit
from AppKit import *
from Foundation import NSObject
from Foundation import NSTimer
from PyObjCTools import AppHelper
class NSImageTest(NSObject):
def init(self):
self = super(NSImageTest, self).init()
if self is None:
return None
self.session = None
self.running = True
return self
def captureOutput_didOutputVideoFrame_withSampleBuffer_fromConnection_(self, captureOutput,
videoFrame, sampleBuffer,
connection):
self.session.stopRunning() # I just want one frame
# Get a bitmap representation of the frame using CoreImage and Cocoa calls
ciimage = CIImage.imageWithCVImageBuffer_(videoFrame)
rep = NSCIImageRep.imageRepWithCIImage_(ciimage)
bitrep = NSBitmapImageRep.alloc().initWithCIImage_(ciimage)
bitdata = bitrep.representationUsingType_properties_(NSBMPFileType, objc.NULL)
# Save image to disk using Cocoa
t0 = time.time()
bitdata.writeToFile_atomically_("grab.bmp", False)
t1 = time.time()
print "Cocoa saved in %.5f seconds" % (t1-t0)
# Save a read-only buffer of image to disk using Python
t0 = time.time()
bitbuf = bitdata.bytes()
f = open("python.bmp", "w")
f.write(bitbuf)
f.close()
t1 = time.time()
print "Python saved buffer in %.5f seconds" % (t1-t0)
# Save a string-copy of the buffer to disk using Python
t0 = time.time()
bitbufstr = str(bitbuf)
f = open("python2.bmp", "w")
f.write(bitbufstr)
f.close()
t1 = time.time()
print "Python saved string in %.5f seconds" % (t1-t0)
# Will exit on next execution of quitMainLoop_()
self.running = False
def quitMainLoop_(self, aTimer):
# Stop the main loop after one frame is captured. Call rapidly from timer.
if not self.running:
AppHelper.stopEventLoop()
def startImageCapture(self, aTimer):
error = None
print "Finding camera"
# Create a QT Capture session
self.session = QTKit.QTCaptureSession.alloc().init()
# Find iSight device and open it
dev = QTKit.QTCaptureDevice.defaultInputDeviceWithMediaType_(QTKit.QTMediaTypeVideo)
print "Device: %s" % dev
if not dev.open_(error):
print "Couldn't open capture device."
return
# Create an input instance with the device we found and add to session
input = QTKit.QTCaptureDeviceInput.alloc().initWithDevice_(dev)
if not self.session.addInput_error_(input, error):
print "Couldn't add input device."
return
# Create an output instance with a delegate for callbacks and add to session
output = QTKit.QTCaptureDecompressedVideoOutput.alloc().init()
output.setDelegate_(self)
if not self.session.addOutput_error_(output, error):
print "Failed to add output delegate."
return
# Start the capture
print "Initiating capture..."
self.session.startRunning()
def main(self):
# Callback that quits after a frame is captured
NSTimer.scheduledTimerWithTimeInterval_target_selector_userInfo_repeats_(0.1,
self,
'quitMainLoop:',
None,
True)
# Turn on the camera and start the capture
self.startImageCapture(None)
# Start Cocoa's main event loop
AppHelper.runConsoleEventLoop(installInterrupt=True)
print "Frame capture completed."
if __name__ == "__main__":
test = NSImageTest.alloc().init()
test.main()
QTKit is deprecated and PyObjC is a big dependency (and seems to be tricky to build if you want it in HomeBrew). Plus PyObjC did not have most of AVFoundation so I created a simple camera extension for Python that uses AVFoundation to record a video or snap a picture, it requires no dependencies (Cython intermediate files are committed to avoid the need to have Cython for most users).
It should be possible to build it like this:
pip install -e git+https://github.com/dashesy/pyavfcam.git
Then we can use it to take a picture:
import pyavfcam
# Open the default video source
cam = pyavfcam.AVFCam(sinks='image')
frame = cam.snap_picture('test.jpg') # frame is a memory buffer np.asarray(frame) can retrieve
Not related to this question, but if the AVFCam class is sub-classed, the overridden methods will be called with the result.

Categories

Resources