Develop Reference

Why did python subprocess stdin.write() not work after a stdin.flush()?

gameserver.py
import config
import os
from subprocess import Popen, PIPE, STDOUT
class GameserverHandler:
def __init__(self):
print("==== Gameserver ====")
self.__gameServerInstance__ = self.restart_gameserver()
print("\tGameserver started")
def restart_gameserver(self):
if os.path.exists(config.bot_config[0]['GameServer']['OutPutFile']):
os.remove(config.bot_config[0]['GameServer']['OutPutFile'])
f = open(config.bot_config[0]['GameServer']['OutPutFile'], "wb")
return Popen(config.bot_config[0]['GameServer']['Path'], stdin=PIPE, stdout=f, stderr=STDOUT)
def send_command(self, command):
try:
self.__gameServerInstance__.stdin.write(str.encode(command))
self.__gameServerInstance__.stdin.flush()
except BrokenPipeError:
pass
except OSError as e:
exit()
main.py
import config
import gameserver
gameServer = gameserver.GameserverHandler()
a = input()
gameServer.send_command('quit\n')
Hallo everyone, i just wrote my first python script.
This script just starts a gameserver on my computer, write the stdout and stderr in a file and gives me an option to send commands to the server.
But there is the Problem when i use send_command() the gameserver dont gets the stdin.write. I read that i have to put a flush() after it. But this is not helping.
Funny is when i change the code to this:
def send_command(self, command):
try:
self.__gameServerInstance__.stdin.write(str.encode(command))
self.__gameServerInstance__.stdin.flush()
self.__gameServerInstance__.stdout.flush()
self.__gameServerInstance__.stderr.flush()
except BrokenPipeError:
pass
except OSError as e:
exit()
I get this Error
Traceback (most recent call last): File "D:\Projekte\python\PycharmProjects\ServerLauncher\main.py", line 7, in <module>
gameServer.send_command('quit') File "D:\Projekte\python\PycharmProjects\ServerLauncher\gameserver.py", line 22, in send_command
self.__gameServerInstance__.stdout.flush() AttributeError: 'NoneType' object has no attribute 'flush'
I think its because i set the stdout, stderr to a file, but why is it working than.
Sorry for a maybe dumb question, i'm just started python programming
UPDATE AFTER ANSWER from Serge Ballesta
Changed the code:
gameserver.py
def restart_gameserver(self):
if os.path.exists(config.bot_config[0]['GameServer']['OutPutFile']):
os.remove(config.bot_config[0]['GameServer']['OutPutFile'])
f = open(config.bot_config[0]['GameServer']['OutPutFile'], "wb")
return Popen(config.bot_config[0]['GameServer']['Path'], stdin=PIPE, stdout=f, stderr=STDOUT)
main.py
import config
import gameserver
import discord
gameServer = gameserver.GameserverHandler()
a = input()
print("SENDING")
gameServer.send_command('quit\n')
print("FINISH")
a = input()
Changes:
stderr to subprocess.STDOUT
change the file operation to 'wb'
add a new line to the send_command
But yeah dont know why but still the process dont get the quit. When i put everything in the main.py and remove the class and so like this.
if os.path.exists(config.bot_config[0]['GameServer']['OutPutFile']):
os.remove(config.bot_config[0]['GameServer']['OutPutFile'])
f = open(config.bot_config[0]['GameServer']['OutPutFile'], "wb")
p = Popen(config.bot_config[0]['GameServer']['Path'], stdin=PIPE, stdout=f, stderr=STDOUT)
a = input()
p.stdin.write(b'quit')
it works, i dont know why, could it be that the stdin is not flushing?
And thx for the fast answer #Serge Ballesta

There are a number of inconsistencies in your code:
you use the same file for both stdout and stderr. That is wrong and may lead to incorrect output in the file. You should use the special value subprocess.STDOUT:
from subprocess import Popen, PIPE, STDOUT
...
return Popen(config.bot_config[0]['GameServer']['Path'], stdin=PIPE,
stdout=f, stderr=STDOUT)
You define the child process with bytes IO, yet open the output file as text. You should use binary mode:
f = open(config.bot_config[0]['GameServer']['OutPutFile'], "wb")
You send a command quit. Most CLI programs expect a command to be terminated with a newline character. You should add a \n to your command:
self.__gameServerInstance__.stdin.write(str.encode(command) + 'b\n')
or
gameServer.send_command('quit\n')
After those fixes, I could successfully start a cmd.exe child process (on Windows), have it to terminate after the exit\n command, and got the expected data in the output file.

mkdir and touch does not work in the same subprocess

import subprocess
def execCommand(cmd):
try:
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate()
return proc.returncode, out, err
except:
print "failed"
return -1, '', ''
if __name__ == '__main__':
res, out, error = execCommand(["mkdir", "-p", "/tmp/newdir/", "&&", "touch", "/tmp/newdir/myfile.txt"])
The touch command above generates myfile.txt as a directory instead of file, could someone explain why?

Don't use subprocess at all for this. The external commands you are running wrap the same library calls that Python can call itself.
import os
os.makedirs("/tmp/newdir")
with open("/tmp/newdir/myfile.txt", "w"):
pass
If you really want a temporary file, consider using the tempfile module instead to create the file securely.

Calling a python script with arguments using subprocess

I have a python script which call another python script from another directory. To do that I used subprocess.Popen :
import os
import subprocess
arg_list = [project, profile, reader, file, str(loop)]
where all args are string if not converted implicitely
f = open(project_path + '/log.txt','w')
proc = subprocess.Popen([sys.executable, python_script] + arg_list, stdin=subprocess.PIPE, stdout=f, stderr=f)
streamdata = proc.communicate()[0]
retCode = proc.returncode
f.close()
This part works well, because of the log file I can see errors that occurs on the called script. Here's the python script called:
import time
import csv
import os
class loading(object):
def __init__(self, project=None, profile=None, reader=None, file=None, loop=None):
self.project=project
self.profile=profile
self.reader=reader
self.file=file
self.loop=loop
def csv_generation(self):
f=open(self.file,'a')
try:
writer=csv.writer(f)
if self.loop==True:
writer.writerow((self.project,self.profile,self.reader))
else:
raise('File already completed')
finally:
file.close()
def main():
p = loading(project, profile, reader, file, loop)
p.csv_generation()
if __name__ == "__main__":
main()
When I launch my subprocess.Popen, I have an error from the called script which tell me that 'project' is not defined. It looks the Popen method doesn't pass arguments to that script. I think i'm doing something wrong, someone has an idea ?

When you pass parameters to a new process they are passed positionally, the names from the parent process do not survive, only the values. You need to add:
import sys
def main():
if len(sys.argv) == 6:
project, profile, reader, file, loop = sys.argv[1:]
else:
raise ValueError,("incorrect number of arguments")
p = loading(project, profile, reader, file, loop)
p.csv_generation()
We are testing the length of sys.argv before the assignment (the first element is the name of the program).

Using input redirection with Popen in python

I need to use stream redirectiton in Popen call in python to use bat file with wine. I need make this:
wine32 cmd < file.bat
It works when I run it manually from terminal, however when I try to call it from python:
proc = Popen('wine32 cmd < file.bat',stdout = PIPE)
I got error: No such file or directory
How to manage with that?
Thanks

Try this:
import sys
#...
with open('file.bat', 'r') as infile:
subprocess.Popen(['wine32', 'cmd'],
stdin=infile, stdout=sys.stdout, stderr=sys.stderr)
Make sure that each argument to wine32 is a separate list element.

maybe you can check this thread.. https://stackoverflow.com/a/5469427/3445802
from subprocess import Popen
p = Popen("batch.bat", cwd=r"C:\Path\to\batchfolder")
stdout, stderr = p.communicate()

Store output of subprocess.Popen call in a string [duplicate]

This question already has answers here:
Running shell command and capturing the output
(21 answers)
Closed 1 year ago.
I'm trying to make a system call in Python and store the output to a string that I can manipulate in the Python program.
#!/usr/bin/python
import subprocess
p2 = subprocess.Popen("ntpq -p")
I've tried a few things including some of the suggestions here:
Retrieving the output of subprocess.call()
but without any luck.

In Python 2.7 or Python 3
Instead of making a Popen object directly, you can use the subprocess.check_output() function to store output of a command in a string:
from subprocess import check_output
out = check_output(["ntpq", "-p"])
In Python 2.4-2.6
Use the communicate method.
import subprocess
p = subprocess.Popen(["ntpq", "-p"], stdout=subprocess.PIPE)
out, err = p.communicate()
out is what you want.
Important note about the other answers
Note how I passed in the command. The "ntpq -p" example brings up another matter. Since Popen does not invoke the shell, you would use a list of the command and options—["ntpq", "-p"].

This worked for me for redirecting stdout (stderr can be handled similarly):
from subprocess import Popen, PIPE
pipe = Popen(path, stdout=PIPE)
text = pipe.communicate()[0]
If it doesn't work for you, please specify exactly the problem you're having.

Python 2: http://docs.python.org/2/library/subprocess.html#subprocess.Popen
from subprocess import PIPE, Popen
command = "ntpq -p"
process = Popen(command, stdout=PIPE, stderr=None, shell=True)
output = process.communicate()[0]
print output
In the Popen constructor, if shell is True, you should pass the command as a string rather than as a sequence. Otherwise, just split the command into a list:
command = ["ntpq", "-p"]
process = Popen(command, stdout=PIPE, stderr=None)
If you need to read also the standard error, into the Popen initialization, you should set stderr to PIPE or STDOUT:
command = "ntpq -p"
process = subprocess.Popen(command, stdout=PIPE, stderr=PIPE, shell=True)
output, error = process.communicate()
NOTE: Starting from Python 2.7, you could/should take advantage of subprocess.check_output (https://docs.python.org/2/library/subprocess.html#subprocess.check_output).
Python 3: https://docs.python.org/3/library/subprocess.html#subprocess.Popen
from subprocess import PIPE, Popen
command = "ntpq -p"
with Popen(command, stdout=PIPE, stderr=None, shell=True) as process:
output = process.communicate()[0].decode("utf-8")
print(output)
NOTE: If you're targeting only versions of Python higher or equal than 3.5, then you could/should take advantage of subprocess.run (https://docs.python.org/3/library/subprocess.html#subprocess.run).

In Python 3.7+ you can use the new capture_output= keyword argument for subprocess.run:
import subprocess
p = subprocess.run(["echo", "hello world!"], capture_output=True, text=True)
assert p.stdout == 'hello world!\n'

Assuming that pwd is just an example, this is how you can do it:
import subprocess
p = subprocess.Popen("pwd", stdout=subprocess.PIPE)
result = p.communicate()[0]
print result
See the subprocess documentation for another example and more information.

for Python 2.7+ the idiomatic answer is to use subprocess.check_output()
You should also note the handling of arguments when invoking a subprocess, as it can be a little confusing....
If args is just single command with no args of its own (or you have shell=True set), it can be a string. Otherwise it must be a list.
for example... to invoke the ls command, this is fine:
from subprocess import check_call
check_call('ls')
so is this:
from subprocess import check_call
check_call(['ls',])
however, if you want to pass some args to the shell command, you can't do this:
from subprocess import check_call
check_call('ls -al')
instead, you must pass it as a list:
from subprocess import check_call
check_call(['ls', '-al'])
the shlex.split() function can sometimes be useful to split a string into shell-like syntax before creating a subprocesses...
like this:
from subprocess import check_call
import shlex
check_call(shlex.split('ls -al'))

This works perfectly for me:
import subprocess
try:
#prints results and merges stdout and std
result = subprocess.check_output("echo %USERNAME%", stderr=subprocess.STDOUT, shell=True)
print result
#causes error and merges stdout and stderr
result = subprocess.check_output("copy testfds", stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError, ex: # error code <> 0
print "--------error------"
print ex.cmd
print ex.message
print ex.returncode
print ex.output # contains stdout and stderr together

This was perfect for me.
You will get the return code, stdout and stderr in a tuple.
from subprocess import Popen, PIPE
def console(cmd):
p = Popen(cmd, shell=True, stdout=PIPE)
out, err = p.communicate()
return (p.returncode, out, err)
For Example:
result = console('ls -l')
print 'returncode: %s' % result[0]
print 'output: %s' % result[1]
print 'error: %s' % result[2]

The accepted answer is still good, just a few remarks on newer features. Since python 3.6, you can handle encoding directly in check_output, see documentation. This returns a string object now:
import subprocess
out = subprocess.check_output(["ls", "-l"], encoding="utf-8")
In python 3.7, a parameter capture_output was added to subprocess.run(), which does some of the Popen/PIPE handling for us, see the python docs :
import subprocess
p2 = subprocess.run(["ls", "-l"], capture_output=True, encoding="utf-8")
p2.stdout

I wrote a little function based on the other answers here:
def pexec(*args):
return subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0].rstrip()
Usage:
changeset = pexec('hg','id','--id')
branch = pexec('hg','id','--branch')
revnum = pexec('hg','id','--num')
print('%s : %s (%s)' % (revnum, changeset, branch))

import os
list = os.popen('pwd').read()
In this case you will only have one element in the list.

import subprocess
output = str(subprocess.Popen("ntpq -p",shell = True,stdout = subprocess.PIPE,
stderr = subprocess.STDOUT).communicate()[0])
This is one line solution

The following captures stdout and stderr of the process in a single variable. It is Python 2 and 3 compatible:
from subprocess import check_output, CalledProcessError, STDOUT
command = ["ls", "-l"]
try:
output = check_output(command, stderr=STDOUT).decode()
success = True
except CalledProcessError as e:
output = e.output.decode()
success = False
If your command is a string rather than an array, prefix this with:
import shlex
command = shlex.split(command)

Use check_output method of subprocess module
import subprocess
address = '192.168.x.x'
res = subprocess.check_output(['ping', address, '-c', '3'])
Finally parse the string
for line in res.splitlines():
Hope it helps, happy coding

For python 3.5 I put up function based on previous answer. Log may be removed, thought it's nice to have
import shlex
from subprocess import check_output, CalledProcessError, STDOUT
def cmdline(command):
log("cmdline:{}".format(command))
cmdArr = shlex.split(command)
try:
output = check_output(cmdArr, stderr=STDOUT).decode()
log("Success:{}".format(output))
except (CalledProcessError) as e:
output = e.output.decode()
log("Fail:{}".format(output))
except (Exception) as e:
output = str(e);
log("Fail:{}".format(e))
return str(output)
def log(msg):
msg = str(msg)
d_date = datetime.datetime.now()
now = str(d_date.strftime("%Y-%m-%d %H:%M:%S"))
print(now + " " + msg)
if ("LOG_FILE" in globals()):
with open(LOG_FILE, "a") as myfile:
myfile.write(now + " " + msg + "\n")