Develop Reference

Python subprocess in .exe

I'm creating a python script that will copy files and folder over the network. it's cross-platform so I make an .exe file using cx_freeze
I used Popen method of the subprocess module
if I run .py file it is running as expected but when i create .exe subprocess is not created in the system
I've gone through all documentation of subprocess module but I didn't find any solution
everything else (I am using Tkinter that also works fine) is working in the .exe accept subprocess.
any idea how can I call subprocess in .exe.file ??
This file is calling another .py file
def start_scheduler_action(self, scheduler_id, scheduler_name, list_index):
scheduler_detail=db.get_scheduler_detail_using_id(scheduler_id)
for detail in scheduler_detail:
source_path=detail[2]
if not os.path.exists(source_path):
showerror("Invalid Path","Please select valid path", parent=self.new_frame)
return
self.forms.new_scheduler.start_scheduler_button.destroy()
#Create stop scheduler button
if getattr(self.forms.new_scheduler, "stop_scheduler_button", None)==None:
self.forms.new_scheduler.stop_scheduler_button = tk.Button(self.new_frame, text='Stop scheduler', width=10, command=lambda:self.stop_scheduler_action(scheduler_id, scheduler_name, list_index))
self.forms.new_scheduler.stop_scheduler_button.grid(row=11, column=1, sticky=E, pady=10, padx=1)
scheduler_id=str(scheduler_id)
# Get python paths
if sys.platform == "win32":
proc = subprocess.Popen(['where', "python"], env=None, stdout=subprocess.PIPE)
else:
proc = subprocess.Popen(['which', "python"], env=None,stdout=subprocess.PIPE)
out, err = proc.communicate()
if err or not out:
showerror("", "Python not found", parent=self.new_frame)
else:
try:
paths = out.split(os.pathsep)
# Create python path
python_path = (paths[len(paths) - 1]).split('\n')[0]
cmd = os.path.realpath('scheduler.py')
#cmd='scheduler.py'
if sys.platform == "win32":
python_path=python_path.splitlines()
else:
python_path=python_path
# Run the scheduler file using scheduler id
proc = subprocess.Popen([python_path, cmd, scheduler_id], env=None, stdout=subprocess.PIPE)
message="Started the scheduler : %s" %(scheduler_name)
showinfo("", message, parent=self.new_frame)
#Add process id to scheduler table
process_id=proc.pid
#showinfo("pid", process_id, parent=self.new_frame)
def get_process_id(name):
child = subprocess.Popen(['pgrep', '-f', name], stdout=subprocess.PIPE, shell=False)
response = child.communicate()[0]
return [int(pid) for pid in response.split()]
print(get_process_id(scheduler_name))
# Add the process id in database
self.db.add_process_id(scheduler_id, process_id)
# Add the is_running status in database
self.db.add_status(scheduler_id)
except Exception as e:
showerror("", e)
And this file is called:
def scheduler_copy():
date= strftime("%m-%d-%Y %H %M %S", localtime())
logFile = scheduler_name + "_"+scheduler_id+"_"+ date+".log"
#file_obj=open(logFile, 'w')
# Call __init__ method of xcopy file
xcopy=XCopy(connection_ip, username , password, client_name, server_name, domain_name)
check=xcopy.connect()
# Cretae a log file for scheduler
file_obj=open(logFile, 'w')
if check is False:
file_obj.write("Problem in connection..Please check connection..!!")
return
scheduler_next_run=schedule.next_run()
scheduler_next_run="Next run at: " +str(scheduler_next_run)
# If checkbox_value selected copy all the file to new directory
if checkbox_value==1:
new_destination_path=xcopy.create_backup_directory(share_folder, destination_path, date)
else:
new_destination_path=destination_path
# Call backup method for coping data from source to destination
try:
xcopy.backup(share_folder, source_path, new_destination_path, file_obj, exclude)
file_obj.write("Scheduler completed successfully..\n")
except Exception as e:
# Write the error message of the scheduler to log file
file_obj.write("Scheduler failed to copy all data..\nProblem in connection..Please check connection..!!\n")
# #file_obj.write("Error while scheduling")
# return
# Write the details of scheduler to log file
file_obj.write("Total skipped unmodified file:")
file_obj.write(str(xcopy.skipped_unmodified_count))
file_obj.write("\n")
file_obj.write("Total skipped file:")
file_obj.write(str(xcopy.skipped_file))
file_obj.write("\n")
file_obj.write("Total copied file:")
file_obj.write(str(xcopy.copy_count))
file_obj.write("\n")
file_obj.write("Total skipped folder:")
file_obj.write(str(xcopy.skipped_folder))
file_obj.write("\n")
# file_obj.write(scheduler_next_run)
file_obj.close()

There is some awkwardness in your source code, but I won't spend time on that. For instance, if you want to find the source_path, it's better to use a for loop with break/else:
for detail in scheduler_detail:
source_path = detail[2]
break # found
else:
# not found: raise an exception
...
Some advice:
Try to separate the user interface code and the sub-processing, avoid mixing the two.
Use exceptions and exception handlers.
If you want portable code: avoid system call (there are no pgrep on Windows).
Since your application is packaged in a virtualenv (I make the assumption cx_freeze does this kind of thing), you have no access to the system-wide Python. You even don't have that on Windows. So you need to use the packaged Python (this is a best practice anyway).
If you want to call a Python script like a subprocess, that means you have two packaged applications: you need to create an exe for the main application and for the scheduler.py script. But, that's not easy to communicate with it.
Another solution is to use multiprocessing to spawn a new Python process. Since you don't want to wait for the end of processing (which may be long), you need to create daemon processes. The way to do that is explained in the multiprocessing module.
Basically:
import time
from multiprocessing import Process
def f(name):
print('hello', name)
if __name__ == '__main__':
p = Process(target=f, args=('bob',))
p.daemon = True
p.start()
# let it live and die, don't call: `p.join()`
time.sleep(1)
Of course, we need to adapt that with your problem.
Here is how I would do that (I removed UI-related code for clarity):
import scheduler
class SchedulerError(Exception):
pass
class YourClass(object):
def start_scheduler_action(self, scheduler_id, scheduler_name, list_index):
scheduler_detail = db.get_scheduler_detail_using_id(scheduler_id)
for detail in scheduler_detail:
source_path = detail[2]
break
else:
raise SchedulerError("Invalid Path", "Missing source path", parent=self.new_frame)
if not os.path.exists(source_path):
raise SchedulerError("Invalid Path", "Please select valid path", parent=self.new_frame)
p = Process(target=scheduler.scheduler_copy, args=('source_path',))
p.daemon = True
p.start()
self.db.add_process_id(scheduler_id, p.pid)
To check if your process is still running, I recommend you to use psutil. It's really a great tool!
You can define your scheduler.py script like that:
def scheduler_copy(source_path):
...
Multiprocessing vs Threading Python
Quoting this answer: https://stackoverflow.com/a/3044626/1513933
The threading module uses threads, the multiprocessing module uses processes. The difference is that threads run in the same memory space, while processes have separate memory. This makes it a bit harder to share objects between processes with multiprocessing. Since threads use the same memory, precautions have to be taken or two threads will write to the same memory at the same time. This is what the global interpreter lock is for.
Here, the advantage of multiprocessing over multithreading is that you can kill (or terminate) a process; you can't kill a thread. You may need psutil for that.

This is not an exact solution you are looking for, but following suggestion should be preferred for two reasons.
These are more pythonic way
subprocess is slightly expensive
Suggestions you can consider
Don't use subprocess for fetching system path. Try check os.getenv('PATH') to get env variable & try to find if python is in the path. For windows, one has to manually add Python path or else you can directly check in Program Files I guess
For checking process ID's you can try psutils. A wonderful answer is provided here at how do I get the process list in Python?
Calling another script from a python script. This does not look cool. Not bad, but I would not prefer this at all.
In above code, line - if sys.platform == "win32": has same value in if and else condition ==> you dont need a conditional statement here.
You wrote pretty fine working code to tell you. Keep Coding!

If you want to run a subprocess in an exe file, then you can use
import subprocess
program=('example')
arguments=('/command')
subprocess.call([program, arguments])

python-daemon to start an independent process but let the main application continue?

Okay I'm officially out of ideas after running each and every sample I could find on google up to 19th page. I have a "provider" script. The goal of this python script is to start up other services that run indefinitely even after this "provider" stopped running. Basically, start the process then forget about it but continue the script and not stopping it...
My problem: python-daemon... I have actions (web-service calls to start/stop/get status from the started services). I create the start commands on the fly and perform variable substitution on the config files as required.
Let's start from this point: I have a command to run (A bash script that executes a java process - a long running service that will be stopped sometime later).
def start(command, working_directory):
pidfile = os.path.join(working_directory, 'application.pid')
# I expect the pid of the started application to be here. The file is not created. Nothing is there.
context = daemon.DaemonContext(working_directory=working_directory,
pidfile=daemon.pidfile.PIDLockFile(pidfile))
with context:
psutil.Popen(command)
# This part never runs. Even if I put a simple print statement at this point, that never appears. Debugging in pycharms shows that my script returns with 0 on with context
with open(pidfile, 'r') as pf:
pid = pf.read()
return pid
From here on in my caller to this method I prepare a json object to return to the client which essentially contains an instance_id (don't mind it) and a pid (that'll be used to stop this process in another request.
What happens? After with context my application exits with status 0, nothing is returned, no json response gets created, no pidfile gets created only the executed psutil.Popen command runs. How can I achieve what I need? I need an independently running process and need to know its PID in order to stop it later on. The executed process must run even if the current python script stops for some reason. I can't get around the shell script as that application is not mine I have to use what I have.
Thanks for any tip!
#Edit:
I tried using simply the Popen from psutil/subprocess with somewhat more promising result.
def start(self, command):
import psutil/subprocess
proc = psutil.Popen(command)
return str(proc.pid)
Now If I debug the application and wait some undefined time on the return statement everything is working great! The service is running the pid is there, I can stop later on. Then I simply ran the provider without debugging. It returns the pid but the process is not running. Seems like Popen has no time to start the service because the whole provider stops faster.
#Update:
Using os.fork:
#staticmethod
def __start_process(command, working_directory):
pid = os.fork()
if pid == 0:
os.chdir(working_directory)
proc = psutil.Popen(command)
with open('application.pid', 'w') as pf:
pf.write(proc.pid)
def start(self):
...
__start_process(command, working_directory)
with open(os.path.join(working_directory, 'application.pid'), 'r') as pf:
pid = int(pf.read())
proc = psutil.Process(pid)
print("RUNNING" if proc.status() == psutil.STATUS_RUNNING else "...")
After running the above sample, RUNNING is written on console. After the main script exits because I'm not fast enough:
ps auxf | grep
No instances are running...
Checking the pidfile; sure it's there it was created
cat /application.pid
EMPTY 0bytes

From multiple partial tips i got, finally managed to get it working...
def start(command, working_directory):
pid = os.fork()
if pid == 0:
os.setsid()
os.umask(0) # I'm not sure about this, not on my notebook at the moment
os.execv(command[0], command) # This was strange as i needed to use the name of the shell script twice: command argv[0] [args]. Upon using ksh as command i got a nice error...
else:
with open(os.path.join(working_directory, 'application.pid'), 'w') as pf:
pf.write(str(pid))
return pid
That together solved the issue. The started process is not a child process of the running python script and won't stop when the script terminates.

Have you tried with os.fork()?
In a nutshell, os.fork() spawns a new process and returns the PID of that new process.
You could do something like this:
#!/usr/bin/env python
import os
import subprocess
import sys
import time
command = 'ls' # YOUR COMMAND
working_directory = '/etc' # YOUR WORKING DIRECTORY
def child(command, directory):
print "I'm the child process, will execute '%s' in '%s'" % (command, directory)
# Change working directory
os.chdir(directory)
# Execute command
cmd = subprocess.Popen(command
, shell=True
, stdout=subprocess.PIPE
, stderr=subprocess.PIPE
, stdin=subprocess.PIPE
)
# Retrieve output and error(s), if any
output = cmd.stdout.read() + cmd.stderr.read()
print output
# Exiting
print 'Child process ending now'
sys.exit(0)
def main():
print "I'm the main process"
pid = os.fork()
if pid == 0:
child(command, working_directory)
else:
print 'A subprocess was created with PID: %s' % pid
# Do stuff here ...
time.sleep(5)
print 'Main process ending now.'
sys.exit(0)
if __name__ == '__main__':
main()
Further info:
Documentation: https://docs.python.org/2/library/os.html#os.fork
Examples: http://www.python-course.eu/forking.php
Another related-question: Regarding The os.fork() Function In Python

python - Using a variable as part of a class method/function

I'm sure this is a simple problem, but I'm an amateur so here i am.
Trying to use the obd python library to communicate with my vehicle. I have a bluetooth OBDII adapter and on it's own i can send single commands as outlined in the basic usage section of the readme and get responses.
My problem is that i have a list of commands that i want to send. I'm trying to use a for loop to go through each command and print the responses on screen.
import os
import time
import obd
def clear_Screen():
os.system('cls' if os.name == 'nt' else 'clear')
connection = obd.OBD("COM7")
pids = [ 'RPM' , 'ENGINE_LOAD' , 'COOLANT_TEMP' , 'RUN_TIME' ]
try:
while True:
time.sleep(1)
clear_Screen()
for i in pids:
cmd = "obd.commands." + i
response = connection.query(cmd)
print i , ": " , (response.value)
except KeyboardInterrupt:
exit()
there's a problem with how I'm building the cmd variable because the response i get from each query is that the command isn't supported but i know they are.
if i print cmd instead of trying to use the command.query(cmd) it prints exactly like it would be if i specified it i.e. obd.commands.RPM so I don't understand why this doesn't work.

In your question you stated:
there's a problem with how I'm building the cmd variable
Indeed, in your way, cmd is only a string. So instead if you want the actual evaluated attitude, you should use getattr instead:
cmd = getattr(obd.commands,i)
eval() and exec() are both unsafe so you should never use those.

Try wrapping what you're assigning to cmd in eval():
cmd = eval("obd.commands." + i)

How to terminate process from Python using pid?

I'm trying to write some short script in python which would start another python code in subprocess if is not already started else terminate terminal & app (Linux).
So it looks like:
#!/usr/bin/python
from subprocess import Popen
text_file = open(".proc", "rb")
dat = text_file.read()
text_file.close()
def do(dat):
text_file = open(".proc", "w")
p = None
if dat == "x" :
p = Popen('python StripCore.py', shell=True)
text_file.write( str( p.pid ) )
else :
text_file.write( "x" )
p = # Assign process by pid / pid from int( dat )
p.terminate()
text_file.close()
do( dat )
Have problem of lacking knowledge to name proces by pid which app reads from file ".proc".
The other problem is that interpreter says that string named dat is not equal to "x" ??? What I've missed ?

Using the awesome psutil library it's pretty simple:
p = psutil.Process(pid)
p.terminate() #or p.kill()
If you don't want to install a new library, you can use the os module:
import os
import signal
os.kill(pid, signal.SIGTERM) #or signal.SIGKILL
See also the os.kill documentation.
If you are interested in starting the command python StripCore.py if it is not running, and killing it otherwise, you can use psutil to do this reliably.
Something like:
import psutil
from subprocess import Popen
for process in psutil.process_iter():
if process.cmdline() == ['python', 'StripCore.py']:
print('Process found. Terminating it.')
process.terminate()
break
else:
print('Process not found: starting it.')
Popen(['python', 'StripCore.py'])
Sample run:
$python test_strip.py #test_strip.py contains the code above
Process not found: starting it.
$python test_strip.py
Process found. Terminating it.
$python test_strip.py
Process not found: starting it.
$killall python
$python test_strip.py
Process not found: starting it.
$python test_strip.py
Process found. Terminating it.
$python test_strip.py
Process not found: starting it.
Note: In previous psutil versions cmdline was an attribute instead of a method.

I wanted to do the same thing as, but I wanted to do it in the one file.
So the logic would be:
if a script with my name is running, kill it, then exit
if a script with my name is not running, do stuff
I modified the answer by Bakuriu and came up with this:
from os import getpid
from sys import argv, exit
import psutil ## pip install psutil
myname = argv[0]
mypid = getpid()
for process in psutil.process_iter():
if process.pid != mypid:
for path in process.cmdline():
if myname in path:
print "process found"
process.terminate()
exit()
## your program starts here...
Running the script will do whatever the script does. Running another instance of the script will kill any existing instance of the script.
I use this to display a little PyGTK calendar widget which runs when I click the clock. If I click and the calendar is not up, the calendar displays. If the calendar is running and I click the clock, the calendar disappears.

So, not directly related but this is the first question that appears when you try to find how to terminate a process running from a specific folder using Python.
It also answers the question in a way(even though it is an old one with lots of answers).
While creating a faster way to scrape some government sites for data I had an issue where if any of the processes in the pool got stuck they would be skipped but still take up memory from my computer. This is the solution I reached for killing them, if anyone knows a better way to do it please let me know!
import pandas as pd
import wmi
from re import escape
import os
def kill_process(kill_path, execs):
f = wmi.WMI()
esc = escape(kill_path)
temp = {'id':[], 'path':[], 'name':[]}
for process in f.Win32_Process():
temp['id'].append(process.ProcessId)
temp['path'].append(process.ExecutablePath)
temp['name'].append(process.Name)
temp = pd.DataFrame(temp)
temp = temp.dropna(subset=['path']).reset_index().drop(columns=['index'])
temp = temp.loc[temp['path'].str.contains(esc)].loc[temp.name.isin(execs)].reset_index().drop(columns=['index'])
[os.system('taskkill /PID {} /f'.format(t)) for t in temp['id']]

Run a .bat program in the background on Windows

I am trying to run a .bat file (which acts as a simulator) in a new window, so it must always be running in the background. I think that creating a new process is the only option that I have. Basically, I want my code to do something like this:
def startSim:
# open .bat file in a new window
os.system("startsim.bat")
# continue doing other stuff here
print("Simulator started")
I'm on Windows so I can't do os.fork.

Use subprocess.Popen (not tested on Windows, but should work).
import subprocess
def startSim():
child_process = subprocess.Popen("startsim.bat")
# Do your stuff here.
# You can terminate the child process after done.
child_process.terminate()
# You may want to give it some time to terminate before killing it.
time.sleep(1)
if child_process.returncode is None:
# It has not terminated. Kill it.
child_process.kill()
Edit: you could also use os.startfile (Windows only, not tested too).
import os
def startSim():
os.startfile("startsim.bat")
# Do your stuff here.

Looks like you want "os.spawn*", which seems to equate to os.fork, but for Windows.
Some searching turned up this example:
# File: os-spawn-example-3.py
import os
import string
if os.name in ("nt", "dos"):
exefile = ".exe"
else:
exefile = ""
def spawn(program, *args):
try:
# check if the os module provides a shortcut
return os.spawnvp(program, (program,) + args)
except AttributeError:
pass
try:
spawnv = os.spawnv
except AttributeError:
# assume it's unix
pid = os.fork()
if not pid:
os.execvp(program, (program,) + args)
return os.wait()[0]
else:
# got spawnv but no spawnp: go look for an executable
for path in string.split(os.environ["PATH"], os.pathsep):
file = os.path.join(path, program) + exefile
try:
return spawnv(os.P_WAIT, file, (file,) + args)
except os.error:
pass
raise IOError, "cannot find executable"
#
# try it out!
spawn("python", "hello.py")
print "goodbye"

On Windows, a background process is called a "service". Check this other question about how to create a Windows service with Python: Creating a python win32 service

import subprocess
proc = subprocess.Popen(['/path/script.bat'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
Using subprocess.Popen() will run the given .bat path ( or any other executable).
If you do wish to wait for the process to finish just add proc.wait():
proc.wait()