Develop Reference

subprocess popen is returning blanck output

On trying to run the grep for the output of previous command using popen returning blank without any error
proc1cmd = "grep " + fetchname1
p1 = subprocess.Popen(['kubectl', 'get', 'abr', '-A'], stdout=subprocess.PIPE)
p2 = subprocess.Popen(proc1cmd, shell=True, stdin=p1.stdout, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p1.stdout.close()
stdout_list = p2.communicate()[0]
stdout_list = stdout_list.decode()
print(p2.stdout)
print(p2.communicate())
output i got:
<_io.BufferedReader name=7>
(b'', b'')

You don't need to concoct a pipeline of kubectl + grep here.
kubectl_output = subprocess.check_output(
["kubectl", "get", "abr", "-A"], encoding="utf-8"
)
matching_lines = [
line for line in kubectl_output.splitlines() if fetchname1 in line
]
print(matching_lines)

How does subprocess.Popen works with shell=False on windows?

I'd like to understand how subprocess.Popen works on windows when shell=False. In particular, how is the environment or variable expansion taken into consideration by the underlying algorithm?
Consider the below simple test:
import os
import subprocess
import textwrap
def truncate(string, width):
if len(string) > width:
string = string[:width - 3] + '...'
return string
if __name__ == '__main__':
index = 0
commands = [
["where", "find.exe"],
"where find.exe",
[r"c:\windows\system32\where.exe", "find.exe"],
r"c:\windows\system32\where.exe find.exe",
["find.exe", "--version"],
"find.exe --version",
r"D:\sources\personal\sublimemerge\Git\usr\bin\find.exe --version",
]
def run(target, shell):
global index
label = (
f'{index:<2}) subprocess.run('
f'{repr(target)}, shell={shell}, '
f'stdout=subprocess.PIPE, stderr=subprocess.STDOUT, '
f'universal_newlines=True)'
)
label = f"{label:<160}"
try:
x = subprocess.run(target, shell=shell, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, universal_newlines=True)
print(f"{label} - returncode={x.returncode} - {repr(truncate(x.stdout, 40))}")
except Exception as e:
print(f"{label} - exception={repr(truncate(str(e), 90))}")
index += 1
for c in commands:
run(c, shell=True)
run(c, shell=False)
Where the output looks like this:
0 ) subprocess.run(['where', 'find.exe'], shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - returncode=0 - 'D:\\sources\\personal\\sublimemerge\\Git\\...'
1 ) subprocess.run(['where', 'find.exe'], shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - returncode=0 - 'D:\\sources\\personal\\sublimemerge\\Git\\...'
2 ) subprocess.run('where find.exe', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - returncode=0 - 'D:\\sources\\personal\\sublimemerge\\Git\\...'
3 ) subprocess.run('where find.exe', shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - returncode=0 - 'D:\\sources\\personal\\sublimemerge\\Git\\...'
4 ) subprocess.run(['c:\\windows\\system32\\where.exe', 'find.exe'], shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - returncode=0 - 'D:\\sources\\personal\\sublimemerge\\Git\\...'
5 ) subprocess.run(['c:\\windows\\system32\\where.exe', 'find.exe'], shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - returncode=0 - 'D:\\sources\\personal\\sublimemerge\\Git\\...'
6 ) subprocess.run('c:\\windows\\system32\\where.exe find.exe', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - returncode=0 - 'D:\\sources\\personal\\sublimemerge\\Git\\...'
7 ) subprocess.run('c:\\windows\\system32\\where.exe find.exe', shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - returncode=0 - 'D:\\sources\\personal\\sublimemerge\\Git\\...'
8 ) subprocess.run(['find.exe', '--version'], shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - returncode=0 - 'find (GNU findutils) 4.6.0\nCopyright ...'
9 ) subprocess.run(['find.exe', '--version'], shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - returncode=2 - 'FIND: Parameter format not correct\n'
10) subprocess.run('find.exe --version', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - returncode=0 - 'find (GNU findutils) 4.6.0\nCopyright ...'
11) subprocess.run('find.exe --version', shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - returncode=2 - 'FIND: Parameter format not correct\n'
12) subprocess.run('D:\\sources\\personal\\sublimemerge\\Git\\usr\\bin\\find.exe --version', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - returncode=0 - 'find (GNU findutils) 4.6.0\nCopyright ...'
13) subprocess.run('D:\\sources\\personal\\sublimemerge\\Git\\usr\\bin\\find.exe --version', shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - returncode=0 - 'find (GNU findutils) 4.6.0\nCopyright ...'
The first question that comes to my mind would be, why do I need to specify the full executable path when shell=False? Why is env not considered the same way than when shell=True is used? Actually... look at cases 5,7, by just looking at those one would think env is being taken into consideration.
I guess the whole matter boils down to understand how env is given to CreateProcess call in cpython's _winapi.c.

Python3 pipe output of multiple processes to a single process [duplicate]

I know how to run a command using cmd = subprocess.Popen and then subprocess.communicate.
Most of the time I use a string tokenized with shlex.split as 'argv' argument for Popen.
Example with "ls -l":
import subprocess
import shlex
print subprocess.Popen(shlex.split(r'ls -l'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE).communicate()[0]
However, pipes seem not to work... For instance, the following example returns noting:
import subprocess
import shlex
print subprocess.Popen(shlex.split(r'ls -l | sed "s/a/b/g"'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE).communicate()[0]
Can you tell me what I am doing wrong please?
Thx

I think you want to instantiate two separate Popen objects here, one for 'ls' and the other for 'sed'. You'll want to pass the first Popen object's stdout attribute as the stdin argument to the 2nd Popen object.
Example:
p1 = subprocess.Popen('ls ...', stdout=subprocess.PIPE)
p2 = subprocess.Popen('sed ...', stdin=p1.stdout, stdout=subprocess.PIPE)
print p2.communicate()
You can keep chaining this way if you have more commands:
p3 = subprocess.Popen('prog', stdin=p2.stdout, ...)
See the subprocess documentation for more info on how to work with subprocesses.

I've made a little function to help with the piping, hope it helps. It will chain Popens as needed.
from subprocess import Popen, PIPE
import shlex
def run(cmd):
"""Runs the given command locally and returns the output, err and exit_code."""
if "|" in cmd:
cmd_parts = cmd.split('|')
else:
cmd_parts = []
cmd_parts.append(cmd)
i = 0
p = {}
for cmd_part in cmd_parts:
cmd_part = cmd_part.strip()
if i == 0:
p[i]=Popen(shlex.split(cmd_part),stdin=None, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=p[i-1].stdout, stdout=PIPE, stderr=PIPE)
i = i +1
(output, err) = p[i-1].communicate()
exit_code = p[0].wait()
return str(output), str(err), exit_code
output, err, exit_code = run("ls -lha /var/log | grep syslog | grep gz")
if exit_code != 0:
print "Output:"
print output
print "Error:"
print err
# Handle error here
else:
# Be happy :D
print output

shlex only splits up spaces according to the shell rules, but does not deal with pipes.
It should, however, work this way:
import subprocess
import shlex
sp_ls = subprocess.Popen(shlex.split(r'ls -l'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
sp_sed = subprocess.Popen(shlex.split(r'sed "s/a/b/g"'), stdin = sp_ls.stdout, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
sp_ls.stdin.close() # makes it similiar to /dev/null
output = sp_ls.communicate()[0] # which makes you ignore any errors.
print output
according to help(subprocess)'s
Replacing shell pipe line
-------------------------
output=`dmesg | grep hda`
==>
p1 = Popen(["dmesg"], stdout=PIPE)
p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
output = p2.communicate()[0]
HTH

"""
Why don't you use shell
"""
def output_shell(line):
try:
shell_command = Popen(line, stdout=PIPE, stderr=PIPE, shell=True)
except OSError:
return None
except ValueError:
return None
(output, err) = shell_command.communicate()
shell_command.wait()
if shell_command.returncode != 0:
print "Shell command failed to execute"
return None
return str(output)

Thank #hernvnc, #glglgl, and #Jacques Gaudin for the answers. I fixed the code from #hernvnc. His version will cause hanging in some scenarios.
import shlex
from subprocess import PIPE
from subprocess import Popen
def run(cmd, input=None):
"""Runs the given command locally and returns the output, err and exit_code."""
if "|" in cmd:
cmd_parts = cmd.split('|')
else:
cmd_parts = []
cmd_parts.append(cmd)
i = 0
p = {}
for cmd_part in cmd_parts:
cmd_part = cmd_part.strip()
if i == 0:
if input:
p[i]=Popen(shlex.split(cmd_part),stdin=PIPE, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=None, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=p[i-1].stdout, stdout=PIPE, stderr=PIPE)
i = i +1
# close the stdin explicitly, otherwise, the following case will hang.
if input:
p[0].stdin.write(input)
p[0].stdin.close()
(output, err) = p[i-1].communicate()
exit_code = p[0].wait()
return str(output), str(err), exit_code
# test case below
inp = b'[ CMServer State ]\n\nnode node_ip instance state\n--------------------------------------------\n1 linux172 10.90.56.172 1 Primary\n2 linux173 10.90.56.173 2 Standby\n3 linux174 10.90.56.174 3 Standby\n\n[ ETCD State ]\n\nnode node_ip instance state\n--------------------------------------------------\n1 linux172 10.90.56.172 7001 StateFollower\n2 linux173 10.90.56.173 7002 StateLeader\n3 linux174 10.90.56.174 7003 StateFollower\n\n[ Cluster State ]\n\ncluster_state : Normal\nredistributing : No\nbalanced : No\ncurrent_az : AZ_ALL\n\n[ Datanode State ]\n\nnode node_ip instance state | node node_ip instance state | node node_ip instance state\n------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n1 linux172 10.90.56.172 6001 P Standby Normal | 2 linux173 10.90.56.173 6002 S Primary Normal | 3 linux174 10.90.56.174 6003 S Standby Normal'
cmd = "grep -E 'Primary' | tail -1 | awk '{print $3}'"
run(cmd, input=inp)

Python multiple stderr pipeing with subprocess.Popen

I'm trying to pass down the error pip but I think it's getting lost
some ware. Here's my code:
from subprocess import *
p = Popen(['ps', '-f', '-u', 'root'], stdout=PIPE, stderr=PIPE)
p1 = Popen(['grep', 'top'], stdin=p.stdout, stdout=PIPE, stderr=p.stderr)
p2 = Popen(['grep', '-v', 'grep'], stdin=p1.stdout,
stdout=PIPE, stderr=p1.stderr)
p3 = Popen(["awk", "{print $8}"], stdin=p2.stdout, stdout=PIPE,
stderr=p2.stderr)
out_v, err_v = p3.communicate()
I don't seem to be able to get it right, what is it that I'm doing wrong ?

iterate over multiple nested results using subprocess

Ok. I've got two shell commands that return string results, which I then need to process further. The first command produces a result (hostnames), and then the next command takes the result of the first, to expand another, more extensive list of hostnames. This is in order to generate a list of hosts to run another shell command on.
I don't need a complete example, just some steering in the right direction.
import subprocess
def getMasterList():
p1 = subprocess.Popen(['get_master_list'], stdout=subprocess.PIPE)
p2 = subprocess.Popen(['grep', '-v', '\['], stdin=p1.stdout, stdout=subprocess.PIPE)
master_list = p2.communicate()[0]
return master_list.split()
def getHostsFromList(master):
p1 = subprocess.Popen(['get_hosts_from_master', master], stdout=subprocess.PIPE)
p2 = subprocess.Popen(['grep', '-v', '\['], stdin=p1.stdout, stdout=subprocess.PIPE)
print p2.communicate()[0]
def doStuff(host):
p = subprocess.Popen('run_shell_command', shell=True, stdout=subprocess.PIPE)
print p.stdout.read()
for master in getMasterList():
getHostsFromList(master)
# How do I iterate further to run doStuff() over the hosts from getHostsFromList(master)?

maybe i'm missing something, but isn't it just:
def getHostsFromList(master):
p1 = subprocess.Popen(['get_hosts_from_master', master], stdout=subprocess.PIPE)
p2 = subprocess.Popen(['grep', '-v', '\['], stdin=p1.stdout, stdout=subprocess.PIPE)
return p2.communicate()[0]
for master in getMasterList():
for host in getHostsFromList(master):
doStuff(host)