Related
In my program, I use subprocess.call to run apache drill automatically.
After that, I make some queries and I would like to print the result.
Before to program the code to run apache drill automatically, I was doing it manually and i could print the results but now i cannot do it.
My last try was to write in a file, but the behavior is the same, nothing is written.
My code is bellow.
import subprocess
from pydrill.client import PyDrill
import sys
writer = open('resultado.txt', 'w')
cmdmsg = subprocess.check_output("cd C:\\Users\\Tito\\Downloads\\apache-drill-1.14.0\\bin & sqlline -u \"jdbc:drill:zk=local\"", shell = True)
writer.write("teste de msg: " + str(cmdmsg))
drill = PyDrill(host='localhost', port=8047)
if drill.is_active:
result = drill.query('''SELECT * FROM cp.`employee.json` LIMIT 3''')
result2 = drill.query('''SELECT * FROM dfs.`/Users/Tito/Desktop/banco_gal.csv` LIMIT 5''')
for tuple in result2:
writer.write(tuple)
writer.close
I could solve this problem.
3 things are important for this topic.
a) we should kill the java virtual machine and shell after that apache drill is turned on.
b) the windows buffer is very short, so the result has not been printed.
c) popen method is better than call for this task.
import os
import re
import subprocess
import traceback
from os import path
from pydrill.client import PyDrill
DRILL_HOST = 'localhost'
DRILL_PORT = 8047
JAVA_HOME = ''
JAVA_HOME = JAVA_HOME or ("JAVA_HOME" in os.environ and os.environ["JAVA_HOME"])
JPS_EXE_PATH = path.join(JAVA_HOME, 'bin', 'jps.exe')
KILL_DRILL_JVM = True
def getJvmPID(className):
pid = None
print('Running JPS cmd: %s' % JPS_EXE_PATH)
jpsOutput = subprocess.check_output(JPS_EXE_PATH)
jpsOutput = str(jpsOutput)
se = re.search(r'([0-9]*\s*)' + className, jpsOutput)
if se:
pid = se.group(1)
return pid
def killProcessByPID(pid):
killCmd = ['taskkill', '/f', '/pid', str(pid)]
print('Running taskkill cmd: %s' % killCmd)
killCmdOuput = subprocess.check_output(killCmd, stderr=subprocess.STDOUT)
print(str(killCmdOuput))
def killJvm(className):
pid = getJvmPID(className)
killProcessByPID(pid)
drillBinDir = 'C:/Users/Tito/Downloads/apache-drill-1.14.0/bin'
sqlinePath = path.join(drillBinDir, 'sqlline.bat')
drillCmdList = [sqlinePath , '-u', '"jdbc:drill:zk=local"']
drillCmdStr = " ".join(drillCmdList)
drillConAttempts = 2
while drillConAttempts > 0:
drillConAttempts -= 1
print("Connecting to drill on %s:%d..." % (DRILL_HOST, DRILL_PORT))
try:
drill = PyDrill(host=DRILL_HOST, port=DRILL_PORT)
except:
print("Exception when creating object")
traceback.print_exc()
print("Checking Drill conection...")
try:
if drill.is_active():
print("Connected.")
break
elif drillConAttempts > 0:
print("Could not connect to Drill. Trying to start Drill...")
print("Running cmd '%s > %s'" % (drillCmdStr, os.devnull) )
devNull = open(os.devnull,"w")
cmdProc = subprocess.Popen(drillCmdStr, cwd=drillBinDir, stdout=devNull, stderr=subprocess.STDOUT, shell=True)
print("Started CMD process with PID %d" %(cmdProc.pid))
except:
print("Exception when checking connection")
traceback.print_exc()
if drill.is_active():
result = drill.query('''SELECT * FROM cp.`employee.json` LIMIT 3''')
for resultTuple in result:
print(resultTuple)
if KILL_DRILL_JVM:
print('Killing Drill process...')
killJvm('SqlLine')
I have a python script which will create the branches and uploads the file in Bitbucket but i am trying to get the links from the Bitbucket after creating branches. How can i do that?
from subprocess import PIPE
import subprocess
import os
import getpass
import pandas as pd
import shutil
git_command = ['git', 'status']
current_user = getpass.getuser()
# Assuming every use has same location for their local repository,
# otherwise this should be set manually
repository = r'C:\Users\%s\bb96' % current_user
git_config = subprocess.call(['git', 'config', 'core.autocrlf', 'true'], cwd=repository, stdout=PIPE, stderr=PIPE, shell=True)
#------------------------------------------------------------------------------
git_fetch = subprocess.Popen(['git', 'fetch'], cwd=repository, stdout=PIPE, stderr=PIPE, shell=True)
stdout_f, stderr_f = git_fetch.communicate()
print(stdout_f)
print(stderr_f)
'''
1. Provide path to where all the BSTM are being stored
Provide app_id
Provide schema name
Provide type of the bstms to be created
for ongoing process -> o
for history load -> h
2. Program creates list of branches to be created as follows:
feature/schema-physcial_tab_name-o
3. For each new feature branch creates a directory tree:
appid\schema\table_name\misc\o\bstm, e.g.:
edld_bb96_rdiapid\crz_rdi_ca\table_name\misc\o\bstm
'''
def loading_to_bitbucket(bstm_path, app_id, schema, bstm_type):
# creates list of bstms
feature_list = []
feature_bstm_ref = {}
bstm_list = [f for f in os.listdir(bstm_path) if os.path.isfile(os.path.join(bstm_path, f)) and f[-4:] == ".csv"]
# creates name of future branch based on table physical name
for bstm in bstm_list:
directorytree = ''
df = pd.read_csv(os.path.join(bstm_path, bstm))
for r in range(len(df)):
for c in range(len(df.columns.values.tolist())):
if str(df.iloc[r,c]).strip() == "Target Physical Table Name":
directorytree = os.path.join(repository,app_id,schema,df.iloc[r+1,c].lower(),'misc',bstm_type,'bstm')
feature_list.append("feature/"+schema+"-"+df.iloc[r+1,c].lower()+"-o")
feature_bstm_ref["feature/"+schema+"-"+df.iloc[r+1,c].lower()+"-o"] = [bstm, directorytree]
break
else:
continue
break
# for each new future branch, new branch is created in bitbucket and file is loaded,
# for existing bstm newer version os loaded only if the bstm file was updated
for feature in feature_list:
compare_flag = 0
x = ''
y = ''
next_release = False
#---------
print(" ")
print("Current iteration: " + feature)
git_pull = subprocess.call(['git', 'pull'], cwd=repository, stdout=PIPE, stderr=PIPE, shell=True)
#git_pull = subprocess.Popen(['git', 'pull'], cwd=repository, stdout=PIPE, stderr=PIPE, shell=True)
#stdout_md, stderr_md = git_pull.communicate()
#print(stdout_md)
#print(stderr_md)
if git_pull != 0:
print("GIT PULL didn't succeed, check your git status.")
break
else:
print("GIT PULL ended successfully.")
checkout_dev = subprocess.call(['git', 'checkout', 'dev'], cwd=repository, stdout=PIPE, stderr=PIPE, shell=True)
if checkout_dev != 0:
print("Can\'t checkout DEV branch, check git status")
break
else:
print("Checked out on DEV successfully.")
create_branch = subprocess.Popen(['git', 'checkout', '-b', feature], cwd=repository, stdout=PIPE, stderr=PIPE, shell=True)
stdout_cb, stderr_cb = create_branch.communicate()
#print(str(stdout_cb))
#print(str(stderr_cb))
if str(stderr_cb).find("fatal: A branch named " + "'" + feature + "'" + " already exists.") < 0 :
try:
createdirtree = subprocess.Popen(['mkdir', feature_bstm_ref[feature][1]], cwd=repository, stdout=PIPE, stderr=PIPE, shell=True)
stdout_md, stderr_md = createdirtree.communicate()
print("Created feature branch: " + feature)
except:
print('Error while creating directory tree for ' + feature)
continue
else:
try:
checkout_branch = subprocess.Popen(['git', 'checkout', feature], cwd=repository, stdout=PIPE, stderr=PIPE, shell=True)
stdout_chb, stderr_chb = checkout_branch.communicate()
print("Checked out on branch: " + feature)
except:
print("Error")
break
try:
x = (os.path.join(bstm_path, feature_bstm_ref[feature][0]).replace('\\', '/'))
y = os.path.join(feature_bstm_ref[feature][1],feature_bstm_ref[feature][0]).replace('\\', '/')
next_release = os.path.isfile(y) # values True/False
#print(next_release)
if next_release:
print("Comparing files",)
compare_files = subprocess.Popen(['git', 'diff', '--shortstat', x, y], stdout=PIPE, stderr=PIPE, shell=True)
stdout, stderr = compare_files.communicate()
#print(str(stdout))
#print(str(stderr))
if str(stdout).find('file changed') > 0 :
compare_flag = 1
if compare_flag == 0:
try:
print("Nothing has changed, move to next avalaible feature branch.")
reset_local = subprocess.Popen(['git', 'reset', '--hard', 'origin/'+ feature], cwd=repository, stdout=PIPE, stderr=PIPE, shell=True)
stdout, stderr = reset_local.communicate()
continue
except:
print("Something went wrong.")
break
except:
print("comparing files didn't succeed.")
break
try:
if compare_flag != 0:
print("Newer version found, new version will be loaded")
else:
print("Initial load of file.")
shutil.copy(os.path.join(bstm_path, feature_bstm_ref[feature][0]), feature_bstm_ref[feature][1])
except shutil.Error:
print(shutil.Error)
print ("Unable to copy file." + feature_bstm_ref[feature][0])
break
# Add the file
try:
add_file = subprocess.Popen(["git", "add", "-A"],cwd = repository, stdout=PIPE, stderr=PIPE, shell=True)
stdout, stderr = add_file.communicate()
#print(str(stdout))
#print(str(stderr))
except:
print ("git add error" + feature)
break
# Compile the commit message and then commit.
try:
message = str(input("Provide commit message: "))
#message = "Initial release " + feature_bstm_ref[feature][0][5:-4] #Assuming that each BSTM will have BSTM_ prefix
commit_info = subprocess.Popen(["git", "commit", "-m", message], cwd = repository,stdout=PIPE, stderr=PIPE, shell=True)
stdout, stderr = commit_info.communicate()
#print(str(stdout))
#print(str(stderr))
except:
print ("git commit error" + feature)
break
# Push to the target BSTM. If you have passphrase for your BitBucket,
# in this step a window will prompt out asking you to input the passphrase.
# After you input the passphrase, the upload will be completed for this feature.
try:
push_to_remote = subprocess.Popen(["git", "push", "origin", "-u",feature], cwd = repository, stdout=PIPE, stderr=PIPE, shell=True)
stdout, stderr = push_to_remote.communicate()
#print(str(stdout))
#print(str(stderr))
print ("Git Bucket uploading succeeded for " + feature)
except:
print ("git push error" + feature)
break
checkout_dev = subprocess.call(['git', 'checkout', 'dev'], cwd=repository, stdout=PIPE, stderr=PIPE, shell=True)
print("All features from the list are checked/loaded.")
#------------------------------------------------------------------------------
bstm_path = r'M:\EDW\EDW_old\Projects\'
app_id = 'dummy'
schema = 'dummy_schema'
bstm_type = 'o'
# calling the function:
loading_to_bitbucket(bstm_path, app_id, schema, bstm_type)....
I have been trying to implement a wrapper around subprocess as follows:
def ans_cmd_stream_color(inputcmd):
"""Driver function for local ansible commands.
Stream stdout to stdout and log file with color.
Runs <inputcmd> via subprocess.
Returns return code, stdout, stderr as dict.
"""
fullcmd = inputcmd
create_debug('Enabling colorful ansible output.', LOGGER)
create_info('Running command: ' + fullcmd, LOGGER, True)
p = subprocess.Popen('export ANSIBLE_FORCE_COLOR=true; ' + fullcmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
stdout_l = []
stderr_l = []
rcode = 0
# Regex black magic
ansi_escape = re.compile(r'\x1b[^m]*m')
# Get the unbuffered IO action going.
try:
# Non blocking
reads = [p.stdout.fileno(), p.stderr.fileno()]
ret = select.select(reads, [], [])
# Print line by line
while True:
for fd in ret[0]:
if fd == p.stdout.fileno():
line = p.stdout.readline()
sys.stdout.write(line.encode('utf-8'))
stdout_l.append(ansi_escape.sub('',
line.encode('utf-8'))
)
if fd == p.stderr.fileno():
line = p.stdout.readline()
sys.stderr.write(line.encode('utf-8'))
stderr_l.append(ansi_escape.sub('',
line.encode('utf-8'))
)
# Break when the process is done.
if p.poll() is not None:
rcode = p.returncode
break
except BaseException as e:
raise e
outstr = ''.join(stdout_l)
errstr = ''.join(stderr_l)
outstr, errstr = str(outstr).rstrip('\n'), str(errstr).rstrip('\n')
expstr = errstr.strip('ERROR: ')
if len(expstr) >= 1:
create_info('Command: ' + str(fullcmd) + ': ' + expstr + '\n', LOGGER,
True)
if rcode == 0:
rcode = 1
else:
create_info(outstr + '\n', LOGGER)
if rcode == 0:
create_info('Command: ' + fullcmd + ' ran successfully.', LOGGER,
True)
expstr = False
ret_dict = {inputcmd: {}}
ret_dict[inputcmd]['rcode'] = rcode
ret_dict[inputcmd]['stdout'] = outstr
ret_dict[inputcmd]['stderr'] = expstr
return copy.deepcopy(ret_dict)
The idea is to print a streaming output of subprocess command and then return info to the function user. The issue is that even using a direct io.open, the subprocess PIP is still buffered unless I set:
os.environ["PYTHONUNBUFFERED"] = "1"
Which is not ideal. Any ideas or has anybody encountered this issue?
UPDATE: With ansible you need to disable buffering for subprocess to honor buffering settings:
def ans_cmd_stream_color(inputcmd):
"""Driver function for local ansible commands.
Stream stdout to stdout and log file with color.
Runs <inputcmd> via subprocess.
Returns return code, stdout, stderr as dict.
"""
fullcmd = inputcmd
create_debug('Enabling colorful ansible output.', LOGGER)
create_info('Running command: ' + fullcmd, LOGGER, True)
p = subprocess.Popen('export ANSIBLE_FORCE_COLOR=true; ' +
'export PYTHONUNBUFFERED=1; ' + fullcmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
stdout_l = []
stderr_l = []
rcode = 0
# Regex black magic
ansi_escape = re.compile(r'\x1b[^m]*m')
# Get the unbuffered IO action going.
try:
# Non blocking
reads = [p.stdout.fileno(), p.stderr.fileno()]
ret = select.select(reads, [], [])
# Print line by line
while True:
for fd in ret[0]:
if fd == p.stdout.fileno():
line = p.stdout.readline()
sys.stdout.write(line.encode('utf-8'))
stdout_l.append(ansi_escape.sub('',
line.encode('utf-8'))
)
if fd == p.stderr.fileno():
line = p.stdout.readline()
sys.stderr.write(line.encode('utf-8'))
stderr_l.append(ansi_escape.sub('',
line.encode('utf-8'))
)
# Break when the process is done.
if p.poll() is not None:
rcode = p.returncode
break
except BaseException as e:
raise e
outstr = ''.join(stdout_l)
errstr = ''.join(stderr_l)
outstr, errstr = str(outstr).rstrip('\n'), str(errstr).rstrip('\n')
expstr = errstr.strip('ERROR: ')
if len(expstr) >= 1:
create_info('Command: ' + str(fullcmd) + ': ' + expstr + '\n', LOGGER,
True)
if rcode == 0:
rcode = 1
else:
create_info(outstr + '\n', LOGGER)
if rcode == 0:
create_info('Command: ' + fullcmd + ' ran successfully.', LOGGER,
True)
expstr = False
ret_dict = {inputcmd: {}}
ret_dict[inputcmd]['rcode'] = rcode
ret_dict[inputcmd]['stdout'] = outstr
ret_dict[inputcmd]['stderr'] = expstr
return copy.deepcopy(ret_dict)
You should probably directly read from the subprocess pipes. Something like the following will read from the standard out to the information logger and the standard error to the error logger.
import logging, subprocess
logging.basicConfig(level=logging.INFO)
proc = subprocess.Popen(
cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
cont = True
while cont:
cont = False
line = proc.stdout.readline()
if not line == b"":
out = line.decode("utf-8").rstrip()
logging.info(out)
cont = True
line = proc.stderr.readline()
if not line == b"":
out = line.decode("utf-8").rstrip()
logging.error(out)
cont = True
if not cont and proc.poll() is not None:
break
To address the buffering issue, per this question, either the subordinate Python script must explicitly flush the buffers, or the environment variable PYTHONUNBUFFERED must be set to a non-empty string.
Here is my code:
def cmdoutput(cmd1, flag):
finish = time.time() + 50
p = subprocess.Popen(cmd1, stdin=subprocess.PIPE, stdout = subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
while p.poll() is None:
time.sleep(1)
if finish < time.time():
os.kill(p.pid, signal.SIGTERM)
print "timed out and killed child, collecting what output exists so far"
if (flag == "1"):#To enable container
out, err = p.communicate(input='container\nzone1')
else:
out, err = p.communicate()
print (out)
return out
When I run this script, I get
Attribute Error: 'module' object has no attribute 'kill'.
What's wrong with my code?
I think you have your own os.py.
Put print os.__file__ before os.kill(...) line, and you will see what's going on.
UPDATE
os.kill is only available in unix in jython
Instead of os.kill(...), use p.kill().
UPDATE
p.kill() not work. (At least in Windows + Jython 2.5.2, 2.5.3).
p.pid is None.
http://bugs.jython.org/issue1898
Change your code as follow. Change CPYTHON_EXECUTABLE_PATH, CMDOUTPUT_SCRIPT_PATH.
CPYTHON_EXECUTABLE_PATH = r'c:\python27\python.exe' # Change path to python.exe
CMDOUTPUT_SCRIPT_PATH = r'c:\users\falsetru\cmdoutput.py' # Change path to the script
def cmdoutput(cmd1, flag):
return subprocess.check_output([CPYTHON_EXECUTABLE_PATH, CMDOUTPUT_SCRIPT_PATH, flag])
Save following code as cmdoutput.py
import subprocess
import sys
def cmdoutput(cmd1, flag):
finish = time.time() + 50
p = subprocess.Popen(cmd1, stdin=subprocess.PIPE, stdout = subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
while p.poll() is None:
time.sleep(1)
if finish < time.time():
p.kill()
return '<<timeout>>'
if flag == "1":
out, err = p.communicate('container\nzone1')
else:
out, err = p.communicate()
return out
if __name__ == '__main__':
cmd, flag = sys.argv[1:3]
print(cmdoutput(cmd, flag))
I'm trying to write a program that will run a program on a remote machine using ssh and pass it SIGINT signals without killing my ssh connection. If I was running on a terminal, then this would be easy with something like
ssh -t -t host "command to run"
I tried this with the following script:
#!/bin/bash
ssh -t -t host "program $#"
and when I run that from the terminal it works fine, but when proofgeneral runs this script and sends it SIGINT it just ends up killing the program (I guess it can't allocate a terminal?).
I've got the following mess of code:
CTRL_C = "<CTRL-C>"
def endpoint(proc, handler):
signal.signal(2, signal.SIG_IGN)
inp = ""
out = ""
err = ""
inp_from_fd = sys.stdin.fileno()
inp_to_fd = proc.stdin.fileno()
out_from_fd = proc.stdout.fileno()
out_to_fd = sys.stdout.fileno()
err_from_fd = proc.stderr.fileno()
err_to_fd = sys.stderr.fileno()
while True:
try:
ins,outs,_ = select.select([inp_from_fd, out_from_fd, err_from_fd],
[inp_to_fd, out_to_fd, err_to_fd],
[])
for fd in ins:
if fd == inp_from_fd:
k = os.read(fd, 1024)
if k == "":
os.close(proc.stdin.fileno())
return
inp = inp + k
elif fd == out_from_fd:
k = os.read(fd, 1024)
out = out + k
elif fd == err_from_fd:
k = os.read(fd, 1024)
err = err + k
else:
assert False
for fd in outs:
if fd == inp_to_fd:
while CTRL_C in inp:
proc.send_signal(2)
p = inp.find(CTRL_C)
inp = inp[0:p] + inp[p+len(CTRL_C):]
k = os.write(fd, inp)
inp = inp[k:]
elif fd == out_to_fd:
k = os.write(fd, out)
out = out[k:]
elif fd == err_to_fd:
k = os.write(fd, err)
err = err[k:]
else:
assert False
except select.error:
pass
except KeyboardInterrupt:
handler()
except IOError, e:
pass
def usage(args):
print "must specify --client or --server"
if __name__ == '__main__':
if len(sys.argv) == 1:
usage(sys.argv)
elif sys.argv[1] == '--server':
proc = subprocess.Popen(sys.argv[2:],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
def INT():
proc.stdin.write(CTRL_C)
proc.stdin.flush()
endpoint(proc, INT)
elif '--client' in sys.argv:
proc = subprocess.Popen(sys.argv[2:],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
import time
time.sleep(1)
def INT():
pass
endpoint(proc, INT)
else:
usage(sys.argv)
which I'm invoking using something like:
remote.py --client ssh -t -t host "remote.py --server <program-to-run> <args>"
Is there something that I'm doing wrong here to handle the signal? I've tried putting a print in the signal 2 handler and it does print it, but it is also killing ssh (I'm getting "Killed by signal 2." printed on the console). Is python forwarding the signal to it's children? Is there a way to get around this? Is there an easier way to do this?
Thanks for any pointers.
os.setpgrp (look at setpgrp manpage) otherwise the signal is propagated to children