Develop Reference

python checking file changes without reading the full file

I have a web app (in the backend) where I am using pysondb (https://github.com/pysonDB/pysonDB) to upload some tasks which will be executed by another program (sniffer).
The sniffer program (a completely separate program) now checks the database for any new unfinished uploaded tasks in an infinite loop and executes them and updates the database.
I don't want to read the database repeatedly, instead want to look for any file changes in the database file (db.json), then read the database only. I have looked into watchdog but was looking for something lightweight and modern to suit my needs.
# infinite loop
import pysondb
import time
from datetime import datetime
# calling aligner with os.system
import os
import subprocess
from pathlib import Path
while True:
# always alive
time.sleep(2)
try:
# process files
db = pysondb.getDb("../tasks_db.json")
tasks = db.getBy({"task_status": "uploaded"})
for task in tasks:
try:
task_path = task["task_path"]
cost = task["cost"]
corpus_folder = task_path
get_output = subprocess.Popen(f"mfa validate {corpus_folder} english english", shell=True, stdout=subprocess.PIPE).stdout
res = get_output.read().decode("utf-8")
# print(type(res))
if "ERROR - There was an error in the run, please see the log." in res:
# log errors
f = open("sniffer_log.error", "a+")
f.write(f"{datetime.now()} :: {str(res)}\n")
f.close()
else:
align_folder = f"{corpus_folder}_aligned"
Path(align_folder).mkdir(parents=True, exist_ok=True)
o = subprocess.Popen(f"mfa align {corpus_folder} english english {align_folder}", shell=True, stdout=subprocess.PIPE).stdout.read().decode("utf-8")
# success
except subprocess.CalledProcessError:
# mfa align ~/mfa_data/my_corpus english english ~/mfa_data/my_corpus_aligned
# log errors
f = open("sniffer_log.error", "a+")
f.write(f"{datetime.now()} :: Files not in right format\n")
f.close()
except Exception as e:
# log errors
f = open("sniffer_log.error", "a+")
f.write(f"{datetime.now()} :: {e}\n")
f.close()

Using python-rq would be a much more efficient way of doing this that wouldn't need a database. It has no requirements other then needing a redis install. From there, you could just move all of that into a function:
def task(task_path, cost):
corpus_folder = task_path
get_output = subprocess.Popen(f"mfa validate {corpus_folder} english english", shell=True, stdout=subprocess.PIPE).stdout
res = get_output.read().decode("utf-8")
# print(type(res))
if "ERROR - There was an error in the run, please see the log." in res:
# log errors
f = open("sniffer_log.error", "a+")
f.write(f"{datetime.now()} :: {str(res)}\n")
... #etc
Obviously you would rename that function and put the try-except statement back, but then you could just call that through RQ:
# ... where you want to call the function
from wherever.you.put.your.task.function import task
result = your_redis_queue.enqueue(task, "whatever", "arguments)

Trying to update command line inputs

My command line input looks like this:
python rerun_edit.py examples/Testing/config.yaml examples/Testing/0.blend examples/Testing/output
And my program looks like this:
import subprocess
import sys
import os
import pathlib
# this sets the amount of scenes
amount_of_scenes = 2
# this sets the amount of runs, which are performed
amount_of_runs = 5
# set the folder in which the run.py is located
rerun_folder = os.path.abspath(os.path.dirname(__file__))
blend_path = ["examples/Testing/0.blend"]
for scene_id in range(amount_of_scenes):
# the first one is the rerun.py script, the last is the output
used_arguments = str(sys.argv[1]) + (blend_path) + str(sys.argv[-1])
output_location = os.path.abspath(sys.argv[-1])
for run_id in range(amount_of_runs):
# in each run, the arguments are reused
cmd = ["python", os.path.join(rerun_folder, "run.py")]
cmd.extend(used_arguments)
# the only exception is the output, which gets changed for each run, so that the examples are not overwritten
#cmd.append(os.path.join(output_location, str(run_id)))
cmd.append(output_location)
print(" ".join(cmd))
# execute one BlenderProc run
subprocess.call(" ".join(cmd), shell=True)
print(used_arguments)
print(cmd)
#get the blend file
old_blend_file = str(scene_id) + ".blend"
new_blend_file = str(scene_id + 1) + ".blend"
blend_path = pathlib.Path(str(blend_path).replace(old_blend_file, new_blend_file))
print(blend_path)
It reads the command line inputs and executes a run.py program for a certain amount of runs.
After the runs I need to update the command line input to the following:
python rerun_edit.py examples/Testing/config.yaml examples/Testing/1.blend examples/Testing/output
So that it executes again for a certain amount of runs but with a different .blend file as input.
I tried to implement a for loop and adjust the paths name after the runs completed but I always getting an error saying:
used_arguments = str(sys.argv[1]) + (blend_path) + str(sys.argv[-1])
TypeError: can only concatenate str (not "list") to str
Can anyone help me out? Any help is highly appreciated.
Thank you very much :)

Here it is
import subprocess
import sys
import os
import pathlib
# this sets the amount of scenes
amount_of_scenes = 2
# this sets the amount of runs, which are performed
amount_of_runs = 5
# set the folder in which the run.py is located
rerun_folder = os.path.abspath(os.path.dirname(__file__))
blend_path = "examples/Testing/0.blend"
for scene_id in range(amount_of_scenes):
# the first one is the rerun.py script, the last is the output
used_arguments = [sys.argv[1], blend_path, sys.argv[-1]]
output_location = os.path.abspath(sys.argv[-1])
for run_id in range(amount_of_runs):
# in each run, the arguments are reused
cmd = ["python", os.path.join(rerun_folder, "run.py")]
cmd += used_arguments
# the only exception is the output, which gets changed for each run, so that the examples are not overwritten
#cmd.append(os.path.join(output_location, str(run_id)))
cmd.append(output_location)
print(" ".join(cmd))
# execute one BlenderProc run
subprocess.call(" ".join(cmd), shell=True)
print(used_arguments)
print(cmd)
blend_path = pathlib.Path(blend_path.replace(("%d.blend" % scene_id), ("%d.blend" % (scene_id + 1))))
print(blend_path)
Please check if it works and comment me if there are any errors

Endless loop when importing a variable from another file in Python

I have a file called main.py and based on the user input I want to run the code either from file export.py, either from file import.py. This is how my code from main.py looks:
print("Hi, " + os.getlogin() + "!")
print("Extracting filenames...")
path = input('Tell me the path!\n')
filenames = [x for x in os.listdir(path)]
ID = [f for f in filenames if re.search("(?<=ID)(\d+)", f)]
filestxt = "Files.txt"
idtxt = "ID.txt"
PathFiles = os.path.join(path, filestxt)
PathID = os.path.join(path, idtxt)
file = open(PathFiles, "w+")
file.write('\n'.join(ID))
file.close()
with open(PathID, 'w') as f:
for item in ID:
list = re.findall("(?<=ID)(\d+)", item)
string = ('\n'.join(list))
f.write("%s\n" % string)
key = input("Export or Import(e/i)?:")
if key == 'e':
os.system('python export.py')
When I am hitting the 'e' button Python is running the code from export.py, but when it gets to the line
from main import PathID
instead of importing the variable which I need for the following function
with open(PathID) as f:
for line in f:
...
the code from main.py is running again and again from the beginning and I get the following lines in the console:
"Hi, " + os.getlogin() + "!"
"Extracting filenames..."
'Tell me the path!\n'
"Export or Import(e/i)?:"
All I want in export.py is to tell Python to read the ID.txt file from the path I have specified in the main.py file.
How can I call the function from main.py in export.py without getting this endless loop?

Try to use
if __name__ == '__main__':
before
key = input ("Export or Import (e / i) ?:")
if key == 'e':
os.system ('python export.py')
or before any code that should always be executed at startup.

To expand on Jossnix's answer:
When you execute os.system('python export.py'), you're launching a separate Python interpreter process.
When you execute from main import PathID within export.py, all of the code in main.py is run, and then, once it's finished running, the control flow is returned to export.py and it has access to PathID. The problem is that, as it stands, your main.py asks for user input. So, your main.py is stuck waiting for user input - you have to provide the input again to this new Python interpreter session! Hence, export.py is stuck while trying to import main.
Jossnix's solution works because it ensures that the user input component of main.py does not get run if main.py is being imported from another module, but will be run if main.py is executed as the main script.
I think you should get rid of the os.system('python export.py') line entirely. It's wasteful: you're launching a completely separate Python interpreter session and the print messages in main.py get run again (this is pretty confusing for the end-user!). I'd say you're better off having whatever code you want to run if the user enters the key 'e' wrapped in a function, and then run this function directly from main.py (if the user has indeed entered 'e'). You could do this: Create such a function (f) in export.py, taking a PathID argument. Then, within main.py, from export import f. Finally, if the user entered 'e', run f(PathID, ...).

Redirecting Output From a Program to a File with Python: Specific Bug

I've been trying to run a Java program and capture it's STDOUT output to a file from the Python script. The idea is to run test files through my program and check if it matches the answers.
Per this and this SO questions, using subprocess.call is the way to go. In the code below, I am doing subprocess.call(command, stdout=f) where f is the file I opened.
The resulted file is empty and I can't quite understand why.
import glob
test_path = '/path/to/my/testfiles/'
class_path = '/path/to/classfiles/'
jar_path = '/path/to/external_jar/'
test_pattern = 'test_case*'
temp_file = 'res'
tests = glob.glob(test_path + test_pattern) # find all test files
for i, tc in enumerate(tests):
with open(test_path+temp_file, 'w') as f:
# cd into directory where the class files are and run the program
command = 'cd {p} ; java -cp {cp} package.MyProgram {tc_p}'
.format(p=class_path,
cp=jar_path,
tc_p=test_path + tc)
# execute the command and direct all STDOUT to file
subprocess.call(command.split(), stdout=f, stderr=subprocess.STDOUT)
# diff is just a lambda func that uses os.system('diff')
exec_code = diff(answers[i], test_path + temp_file)
if exec_code == BAD:
scream(':(')

I checked the docs for subprocess and they recommended using subprocess.run (added in Python 3.5). The run method returns the instance of CompletedProcess, which has a stdout field. I inspected it and the stdout was an empty string. This explained why the file f I tried to create was empty.
Even though the exit code was 0 (success) from the subprocess.call, it didn't mean that my Java program actually got executed. I ended up fixing this bug by breaking down command into two parts.
If you notice, I initially tried to cd into correct directory and then execute the Java file -- all in one command. I ended up removing cd from command and did the os.chdir(class_path) instead. The command now contained only the string to run the Java program. This did the trick.
So, the code looked like this:
good_code = 0
# Assume the same variables defined as in the original question
os.chdir(class_path) # get into the class files directory first
for i, tc in enumerate(tests):
with open(test_path+temp_file, 'w') as f:
# run the program
command = 'java -cp {cp} package.MyProgram {tc_p}'
.format(cp=jar_path,
tc_p=test_path + tc)
# runs the command and redirects it into the file f
# stores the instance of CompletedProcess
out = subprocess.run(command.split(), stdout=f)
# you can access useful info now
assert out.returncode == good_code

Calling a python script with arguments using subprocess

I have a python script which call another python script from another directory. To do that I used subprocess.Popen :
import os
import subprocess
arg_list = [project, profile, reader, file, str(loop)]
where all args are string if not converted implicitely
f = open(project_path + '/log.txt','w')
proc = subprocess.Popen([sys.executable, python_script] + arg_list, stdin=subprocess.PIPE, stdout=f, stderr=f)
streamdata = proc.communicate()[0]
retCode = proc.returncode
f.close()
This part works well, because of the log file I can see errors that occurs on the called script. Here's the python script called:
import time
import csv
import os
class loading(object):
def __init__(self, project=None, profile=None, reader=None, file=None, loop=None):
self.project=project
self.profile=profile
self.reader=reader
self.file=file
self.loop=loop
def csv_generation(self):
f=open(self.file,'a')
try:
writer=csv.writer(f)
if self.loop==True:
writer.writerow((self.project,self.profile,self.reader))
else:
raise('File already completed')
finally:
file.close()
def main():
p = loading(project, profile, reader, file, loop)
p.csv_generation()
if __name__ == "__main__":
main()
When I launch my subprocess.Popen, I have an error from the called script which tell me that 'project' is not defined. It looks the Popen method doesn't pass arguments to that script. I think i'm doing something wrong, someone has an idea ?

When you pass parameters to a new process they are passed positionally, the names from the parent process do not survive, only the values. You need to add:
import sys
def main():
if len(sys.argv) == 6:
project, profile, reader, file, loop = sys.argv[1:]
else:
raise ValueError,("incorrect number of arguments")
p = loading(project, profile, reader, file, loop)
p.csv_generation()
We are testing the length of sys.argv before the assignment (the first element is the name of the program).