import subprocess
def run_cmd(args_list):
print('Running system command: {0}'.format(' '.join(args_list)))
proc = subprocess.Popen(args_list, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
proc.communicate()
return proc.returncode
cmd = ['hadoop', 'fs', '-test', '-e', hdfs_file_path]
code = run_cmd(cmd)
if code:
print 'file not exist'
When I give this command to see whether file exists in HDFS, it is throwing me this error:
RuntimeError: Error running command: hadoop fs -test -f /app/tmp/1.json. Return code: 1, Error: b''
How to resolve this issue?
I would use an api instead of calling subprocesses. It is always better to use an api for this for example snakebite which is created by spotify. This example checks if a file exists in the given folder:
from snakebite.client import Client
client = Client("localhost", 8020, use_trash=False)
return "fileName" in client.ls(['hdfs_path'])
Related
when i try to run a subprocess (in windows 10) that connects to ssh, i get:
Warning: Identity file 'C:\Users\rndom\.ssh\ssh-keys\rndom' not accessible: No such file or directory.
i've tried running the python script from a cmd with admin privileges, but it returns the same error.
i was able to connect with plink, so not sure why the issues here. my goal is to just run an initial connection so that i can get the host key into the known_hosts file.
for that reason i am using Popen:
from remoteSSH import *
import json
from pathlib import Path, PureWindowsPath
import subprocess
import getpass
import time
ssh_connect = rf"ssh -tt -i 'C:\Users\{getpass.getuser()}\.ssh\ssh-keys\{server['HOST_USER']}' {server['HOST_USER']}#{server['HOST_IP']}"
ssh_parsed = shlex.split(ssh_connect)
try:
with Popen(ssh_parsed, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc:
time.sleep(0.5)
proc.stdin.flush()
proc.stdin.write(('yes'+'\n').encode('utf-8')) # needs to be in bytes
proc.stdin.flush()
proc.wait()
proc.stdin.close()
proc.stdout.close()
proc.terminate()
i've also tried running as admin as a test:
run_as_admin = rf'runas /noprofile /user:{getpass.getuser()} notepad.exe\n'
admin_parsed = shlex.split(run_as_admin)
with Popen(admin_parsed , stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc:
doesn't work
edit: i rather not use paramiko or whatever...
I have a batch file which is running a python script and in the python script, I have a subprocess function which is being ran.
I have tried subprocess.check_output, subprocess.run, subprocess.Popen, all of them returns me an empty string only when running it using a batch file.
If I run it manually or using an IDE, I get the response correctly. Below is the code for subprocess.run:
response = subprocess.run(fileCommand, shell=True, cwd=pSetTableauExeDirectory, capture_output=True)
self.writeInLog(' Command Response: \t' + str(response))
Response is in stdout=b''
When ran in batch file and from task scheduler:
Command Response: CompletedProcess(args='tableau refreshextract
--config-file "Z:\XXX\tableau_config\SampleSuperStore.txt"',
returncode=0, stdout=b'', stderr=b'')
When ran manually or in IDE:
Command Response: CompletedProcess(args='tableau refreshextract
--config-file "Z:\XXX\tableau_config\SampleSuperStore.txt"',
returncode=0, stdout=b'Data source refresh completed.\r\n0 rows uploaded.\r\n', stderr=b'')
Batch file which runs the python program. Parameters are parsed to the python application
SET config=SampleSuperStore.txt
CALL C:\XXX\AppData\Local\Continuum\anaconda3\Scripts\activate.bat
C:\XXX\AppData\Local\Continuum\anaconda3\python.exe Z:\XXX\pMainManual.py "%config%"
Why is that??
--Complete python code---
try:
from pWrapper import wrapper
import sys
except Exception as e:
print(str(e))
class main:
def __init__(self):
self.tableauPath = 'C:\\Program Files\\Tableau\\Tableau 2018.3\\bin\\'
self.tableauCommand = 'tableau refreshextract --config-file'
def runJob(self,argv):
self.manual_sProcess(argv[1])
def manual_sProcess(self,tableauConfigFile):
new_wrapper = wrapper()
new_wrapper.tableauSetup(self.tableauPath,self.tableauCommand)
if new_wrapper.tableauConfigExists(tableauConfigFile):
new_wrapper.tableauCommand(tableauConfigFile)
if __name__ == "__main__":
new_main = main()
new_main.runJob(sys.argv)
Wrapper class:
def tableauCommand(self,tableauConfigFile):
command = self.setTableauExeDirectory + ' ' + self.refreshConfigCommand + ' "' + tableauConfigFile + '"'
self.new_automateTableauExtract.runCommand(tableauConfigFile,command,self.refreshConfigCommand,self.tableauFilePath,self.setTableauExeDirectory)
Automate Class:
def runCommand(self,pConfig,pCommand,pRefreshConfigCommand,pFilePath,pSetTableauExeDirectory):
try:
fileCommand = pRefreshConfigCommand + ' "' + pFilePath + '"'
response = subprocess.run(fileCommand, shell=True, cwd=pSetTableauExeDirectory, capture_output=True)
self.writeInLog(' Command Response: \t' + str(response))
except Exception as e:
self.writeInLog('Exception in function runCommand: ' + str(e))
UPDATE: I initially thought that the bat file was causing this issue but it looks like it works when running manually a batch file but not when it is set on task scheduler
Updated
First of all, if there is a need to run anaconda-prompt by calling activate.bat file, you can simply do as follows:
import subprocess
def call_anaconda_venv():
subprocess.call('python -m venv virtual.env')
subprocess.call('cmd.exe /k /path/venv/Scripts/activate.bat')
if __name__ == "__main__":
call_anaconda_venv()
The result of the above code would be a running instance of anaconda-prompt as required.
Now as Problem Seems Like:
I have a batch file which is running a python script and in the python script, I have a subprocess function which is being run.
I have implemented the same program as required; Suppose we have
Batch File ---> script.bat **** includes a command to run python script i.e test.py. ****
Python Script File ---> test.py **** includes a method to run commands using subprocess. ****
Batch File ---> sys_info.bat **** includes a command which would give the system information of my computer. ****
Now First, script.bat includes a command that will run the required python script as given below;
python \file_path\test.py
pause
Here, pause command is used to prevent auto-closing console after execution. Now we have test.py, python script which includes subprocess method to run required commands and get their output.
from subprocess import check_output
class BatchCommands:
#staticmethod
def run_commands_using_subprocess(commands):
print("Running commands from File: {}".format(commands))
value = check_output(commands, shell=True).decode()
return value
#staticmethod
def run():
commands_from_file = "\file-path\sys_info.bat"
print('##############################################################')
print("Shell Commands using >>> subprocess-module <<<")
print('##############################################################')
values = BatchCommands.run_commands_using_subprocess(commands_from_file)
print(values)
if __name__ == '__main__':
BatchCommands.run()
Now, in the end, I have a sys_info.bat file which includes commands to renew the IP-Adress of my computer. Commands in sys_info.bat file are as follows;
systeminfo
Place multiple commands in sys_info.bat file, then you can also run multiple commands at a time like:
ipconfig/all
ipconfig/release
ipconfig/reset
ipconfig/renew
ipconfig
Before to use the file, set all files directory paths, and run the batch file i.e script.py in command-prompt as follows;
Run command-prompt or terminal as an administrator.
run \file_path\script.py
Here is the result after running the batch file in the terminal.
This is happening because your ide is not running in a shell that works in the way that open subprocess is expecting.
If you set SHELL=False and specify the absolute path to the batch file it will run.
you might still need the cwd if the batch file requires it.
I am trying to move a folder into another folder but am getting Permission Denied error when I try to perform this operation in a Python script vs. the move working successfully when I run it in bash or even in Python interactive mode.
cmd = ['sudo', 'mv', '/path1/dir', '/path2']
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if p.returncode != 0:
print(stderr)
I also tried adding shell=True.
p = subprocess.Popen(' '.join(cmd), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if p.returncode != 0:
print(stderr)
In both cases, I am getting the following error:
"mv: cannot move '/path1/dir' to '/path2/dir': Permission denied\n"
I invoke my script in the following manner:
sudo python script.py
I tried executing each command in shell as well as Python interactive mode and I don't get any errors. Any idea what is going on over here?
After wasting hours of time debugging as to what was going wrong, I finally figured out what was happening. I was creating /path1 and /path2 using tempfile. Here is a snippet of the code:
class UtilitiesTest(unittest.TestCase):
#staticmethod
def createTestFiles():
dir = tempfile.mkdtemp()
_, file = tempfile.mkstemp(dir=dir)
return dir, file
def test_MoveFileToAnotherLocation(self):
src_dir, src_file = UtilitiesTest.createTestFiles()
dest_dir, dest_file = UtilitiesTest.createTestFiles()
cmd = ['sudo', 'mv', src_dir, dest_dir]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if p.returncode != 0:
print(stderr)
Like zwer said in the comments, if I am running this script using sudo, I don't need to add sudo in my mv command. Because I kept getting permission denied errors, I kept thinking that sudo would fix my problem. The actual issue over here was when tempfile.mkstemp() is called, it returns an open file descriptor along with the file path. I didn't pay much attention to the first argument, so when I modified my createTestFiles() to below, everything started working.
#staticmethod
def createTestFiles():
dir = tempfile.mkdtemp()
fd, file = tempfile.mkstemp(dir=dir)
os.close(fd)
return dir, file
import subprocess
def execCommand(cmd):
try:
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate()
return proc.returncode, out, err
except:
print "failed"
return -1, '', ''
if __name__ == '__main__':
res, out, error = execCommand(["mkdir", "-p", "/tmp/newdir/", "&&", "touch", "/tmp/newdir/myfile.txt"])
The touch command above generates myfile.txt as a directory instead of file, could someone explain why?
Don't use subprocess at all for this. The external commands you are running wrap the same library calls that Python can call itself.
import os
os.makedirs("/tmp/newdir")
with open("/tmp/newdir/myfile.txt", "w"):
pass
If you really want a temporary file, consider using the tempfile module instead to create the file securely.
I'm running on a machine with python 2.6 and no I can't upgrade for now.
I need the subrpocess.check_output function but as I've understood this is note defined in 2.6.
So I've used a workaround:
try:
import subprocess
if "check_output" not in dir( subprocess ): # duck punch it in!
def check_output(*popenargs, **kwargs):
r"""Run command with arguments and return its output as a byte string.
Backported from Python 2.7 as it's implemented as pure python on stdlib.
>>> check_output(['/usr/bin/python', '--version'])
Python 2.6.2
"""
process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
output, unused_err = process.communicate()
retcode = process.poll()
if retcode:
cmd = kwargs.get("args")
if cmd is None:
cmd = popenargs[0]
error = subprocess.CalledProcessError(retcode, cmd)
error.output = output
raise error
return output
subprocess.check_output = check_output
# Git Information
git_info= {
"last_tag" : subprocess.check_output(['git', 'describe', '--always']),
"last_commit" : subprocess.check_output(['git', 'log', '-1', '--pretty=format:\'%h (%ci)\'', '--abbrev-commit'])
}
except Exception, e:
raise e
else:
data = git_info
return data
I'm using this in conjunction with Django + wsgi.
The previous piece of code always give me Command '['git', 'describe', '--always']' returned non-zero exit status 128.
Now if I run git describe --always I get a correct output so I don't think the problem is there.
I have no idea what could cause the problem.
EDIT:
If I use the command subprocess.check_output(['ls', '-l']) or subprocess.check_output(['pwd']) things work and from here I've understood that the view called from Django is actually running at /var/www being this the DocumentRoot specified in the Apache config file.
The real file is not located under /var/www in fact everything works on my local machine where I use the local django dev server. So the git command won't work because there is no git repository under /var/www. How can I execute the original subprocess.check_output(['git', 'describe', '--always']) from its original path (where the python file is actually located)?
I've solved by passing the cwd argument to check_output as suggested in a comment.
def get_git_info():
git_info = {}
try:
import subprocess
# subprocess.check_output did not exist in 2.6
if "check_output" not in dir(subprocess): # duck punch it in!
# workaround/redefinition for the subprocess.check_output() command
def check_output(*popenargs, **kwargs):
""" Run command with arguments and return its output as a byte string.
Backported from Python 2.7 as it's implemented as pure python on stdlib.
>>> check_output(['/usr/bin/python', '--version'])
Python 2.6.2
"""
process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
output, unused_err = process.communicate()
retcode = process.poll()
if retcode:
cmd = kwargs.get("args")
if cmd is None:
cmd = popenargs[0]
error = subprocess.CalledProcessError(retcode, cmd)
error.output = output
raise error
# In case we want the error in string format:
# stderr=subprocess.STDOUT
# raise Exception(stderr)
return output
subprocess.check_output = check_output
# Set on which dir the git command should be invoked
if os.path.isdir(r'/my/git/dir'):
cwd = r'/my/git/dir'
# If using the django local dev server, then it will invoke the command from the dir where this script is located
else:
cwd = None
# Check that the directory is a git repo:
# 'git rev-parse' returns a number !=0 if we are in a git repo
if subprocess.check_output(['git', 'rev-parse'], cwd=cwd) != 0:
# Git Information
git_info = {
"last_tag": subprocess.check_output(['git', 'describe', '--always'], cwd=cwd),
"last_commit": subprocess.check_output(['git', 'log', '-1', '--pretty=format:\'%h (%ci)\'', '--abbrev-commit'], cwd=cwd),
}
except Exception, e:
log.exception('Problem getting git information')
pass
# return the git info or an empty dict (defined above)
return git_info