For example there's with statement:
with open("ACCELEROMETER", 'w') as ACCELEROMETER,\
open('GPS', 'w') as GPS,\
open('LIGHT', 'w') as LIGHT,\
open('LOCATION', 'w') as LOCATION,\
open('MIC', 'w') as MIC,\
open('SCREEN', 'w') as SCREEN,\
open('TIME', 'w') as TIME:
I want to get file objects just created using some python code :
I'm looking for equivalent of dir function for local scope of with.
Is it possible?
What you're asking for isn't really possible (without making some assumptions) since with doesn't create a new namespace. You could create a file-list object which is implemented as a context manager ...
class FileList(list):
def __init__(self, files, mode='r'):
list.__init__(open(arg, mode) for arg in files)
def __enter__(self):
return self
def __exit__(self, *args):
for fobj in self:
fobj.close()
with FileList(["ACCELEROMETER", "GPS", ...], mode='w') as fl:
for fobj in fl:
...
Related
I have a library function which I ran through a wrapper function to disable printing.
However, I need to somehow make that wrapped function not cause pickling errors. I get AttributeError: Can't pickle local object 'mute.<locals>.wrapper'
First bottom line of the stack File "C:\Users\[...]\Python310\site-packages\neat\checkpoint.py", line [...], in save_checkpoint pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
I am only including my best guess which code is relevant, cause the whole code is quite big.
bu.py
def mute(f):
def wrapper(*args, **kwargs):
import sys
original_stdout = sys.stdout
sys.stdout = open(os.devnull, 'w')
f(*args, **kwargs)
sys.stdout.close()
sys.stdout = original_stdout
return wrapper
main.py
...
p = neat.Population(config)
reporter_to_add = neat.Checkpointer(1)
reporter_to_add.save_checkpoint = bu.mute(reporter_to_add.save_checkpoint)
p.add_reporter(reporter_to_add)
reporter_to_add = custom_reporters.FullPopulationCheckpointer(1)
reporter_to_add.save_checkpoint = bu.mute(reporter_to_add.save_checkpoint)
p.add_reporter(reporter_to_add)
p.run(gym.eval_genomes, 10)
...
neat.Checkpoint.py
class Checkpointer(BaseReporter):
...
def save_checkpoint(self, config, population, species_set, generation):
""" Save the current simulation state. """
filename = '{0}{1}'.format(self.filename_prefix,generation)
print("Saving checkpoint to {0}".format(filename))
with gzip.open(filename, 'w', compresslevel=5) as f:
data = (generation, config, population, species_set, random.getstate())
pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
I have tried putting a global wrapper in the mute() function, but that didn't work and also I don't know if I should have them global, cause there will be multiple save_checkpoint functions as you can see. They are different.
Everything works fine if the bu.mute() wrapper is not used.
I want to read in data from 'path_to_csv_file' and print. A piece of code that works is:
import csv
class MyClass:
def f1(self):
with open('path_to_csv_file', "r") as f:
reader = csv.reader(f)
for row in reader:
print(row)
Now, I would like to define the reader out of my printing function. For this I try:
import csv
class MyClass:
def __init__(self):
with open('path_to_csv_file', "r") as f:
self.reader = csv.reader(f)
def f2(self):
for row in self.reader:
print(row)
But executing f2:
MyClass().f2()
raises the following error:
for row in self.reader:
ValueError: I/O operation on closed file.
How can I define the reader out of my function and yet use it inside? (The reason why I am interested in this is that I will be using the reader in many functions inside MyClass and I think it is cleaner to define it only once.)
The moment, execution comes out of with, it will close the file. if you need the file to be open, you have to open it with open() and assign it to a variable to close once done. Refer below.
import csv
class MyClass:
def __init__(self):
self.file=open('csv_file', "r")
self.reader = csv.reader(self.file)
def f2(self):
for row in self.reader:
print(row)
self.file.close()
MyClass().f2()
As Pranav, noted in comments, If you need the file to be open for other functions or for multiple calls of f1(), keep self.file.close() out of f2(), may be in its own function.
I don't know if there is an easy way of doing this that doesn't rely on manually writing down what the saved outputs from a script are so open to any suggestions.
I want a function that runs at the end of my script and that automatically generates a text file with a name like:
"IO_track_scriptname_date_time"
Which has a list of the files I loaded and the files I saved (location links).
And then saves this txt file to the desired destination.
Thank you for your help
Edit:
Or any alternative way of keeping a log of inputs and outputs.
Here is a thin object wrapper around the open function that tracks all of the files that are opened.
class Open:
_open = open
def __init__(self):
self.opened_files = []
self.fp = None
def __call__(self,
file,
mode='r',
buffering=-1,
encoding=None,
errors=None,
newline=None,
closefd=True,
opener=None):
self.fp = self._open(file, mode, buffering, encoding, errors,
newline, closefd, opener)
self.opened_files.append((mode, file))
return self.fp
def __enter__(self, *args, **kwargs):
return self.__call__(*args, **kwargs)
def __exit__(self, *exc_details):
return self.fp.close()
def __getattr__(self, attr):
return getattr(self.fp, attr)
def export(self, filename):
with open(filename, 'w') as fp:
for m, fn in self.opened_files:
fp.write(f'({m}): {fn}\n')
To actually use it, you will need to overwrite the built-in open function with an instantiation of this class. If you have one file that you are calling, you can pop this into the __main__ block. i.e.
...
if __name__=='__main__':
# code defining Open class here
...
open = Open()
# other code in __main__ here
open.export("IO_track_scriptname_date_time.txt")
The following method:
def generateCSVfile(fileName,fileDescription,fileLocation,md5Hash):
with open('deploymentTemplate.csv', 'w') as csvfile:
createRow = csv.writer(csvfile,
quoting=csv.QUOTE_MINIMAL)
This generates my CSV file but since I am calling it in a loop it just overrides itself.
generateCSVfile(name, fileDescription, filePath+"/"+name, md5Hash)
I am trying to find a way to generate the file, leave it open, call the above method and have all the text written to it without the file overriding itself.
Use : open('deploymentTemplate.csv', 'a') to append values.
Syntax: open(<file_name> [,<mode>])
Different modes are :
mode can be 'r' when the file will only be read
'w' for only writing (an existing file with the same name will be erased)
'a' opens the file for appending and any data written to the file is
automatically added to the end.
'r+' opens the file for both reading and writing.
The mode argument is optional; 'r' will be assumed if it’s omitted.
Eg :
with open("test.txt", "a") as myfile:
myfile.write("appended text")
If the file needs to be emptied once per program run, but appended multiple times within a run, you could always just use a global (or class member state) to ensure it's only opened once.
import atexit
csvfile = None
def generateCSVfile(fileName,fileDescription,fileLocation,md5Hash):
global csvfile
if csvfile is None:
# Lazily open file on first call
csvfile = open('deploymentTemplate.csv', 'w')
atexit.atexit(csvfile.close) # Close cleanly on program exit
try:
csvwriter = csv.writer(csvfile, quoting=csv.QUOTE_MINIMAL, newline='')
# do whatever writing you need to csvwriter
finally:
csvfile.flush() # Match behavior of repeated with/open, force predictable flush
If there might be multiple CSV files involved, you might use a class with instance state and a method to do the writing, so each file can be independently cleared once and appended many times. In this case, due to limits on the number of open file handles, reopening for append on each use is slower but safer than opening once and leaving open. You can use caching so the class is a singleton for any given file name too:
import weakref
class CSVGenerator:
CACHE = {}
CACHELOCK = threading.Lock()
def __new__(cls, csvfilename):
canonicalname = os.path.realpath(csvfilename)
newself = super().__new__(cls)
with cls.CACHELOCK:
self = cls.CACHE.setdefault(canonicalname, newself)
if newself is self:
# First time we opened this file, clear file and initialize instance
with open(canonicalname, 'w') as f:
pass
self.csvfilename = canonicalname
self.appendlock = threading.Lock()
return self
def generateCSVfile(self, fileName, fileDescription, fileLocation, md5Hash):
with newself.appendlock, open(self.csvfilename, 'a', newline='') as csvfile:
createRow = csv.writer(csvfile, quoting=csv.QUOTE_MINIMAL)
# Perform writes to file
Usage of the class can be either:
CSVGenerator(somecsvfilename).generateCSVfile(...args...)
which acquires an instance briefly (creating it if needed) then writes once, or it can create and store an instance and reuse it (saves the overhead of cache lookup, but functionally identical).
I always open and write into files using with statement:
with open('file_path', 'w') as handle:
print >>handle, my_stuff
However, there is one instance where I need to be able to be more flexible, and write to sys.stdout (or other types of streams), if that is provided instead of file path:
So, my question is this: Is there a way for using with statement both with real files and with sys.stdout?
Note that I can use the following code, but I think this defeats the purpose of using with:
if file_path != None:
outputHandle = open(file_path, 'w')
else:
outputHandle = sys.stdout
with outputHandle as handle:
print >>handle, my_stuff
You can create a context manager and use it like this
import contextlib, sys
#contextlib.contextmanager
def file_writer(file_name = None):
# Create writer object based on file_name
writer = open(file_name, "w") if file_name is not None else sys.stdout
# yield the writer object for the actual use
yield writer
# If it is file, then close the writer object
if file_name != None: writer.close()
with file_writer("Output.txt") as output:
print >>output, "Welcome"
with file_writer() as output:
print >>output, "Welcome"
If you don't pass any input to file_writer it will use sys.stdout.
Thing is, you don't need to use a context processor with stdout, because you're not opening or closing it. A less fancy way of abstracting this is:
def do_stuff(file):
# Your real code goes here. It works both with files or stdout
return file.readline()
def do_to_stdout():
return do_stuff(sys.stdout)
def do_to_file(filename):
with open(filename) as f:
return do_stuff(f)
print do_to_file(filename) if filename else do_to_stdout()
The simplest way is to simply use "old school" streamed filenames, that way your code doesn't have to change. In Unix this is "/dev/tty" or in Windows this is "con" (although there are other choices for both platforms).
if default_filename is None:
default_filename = "/dev/tty"
with open(default_filename, 'w') as handle:
handle.write("%s\n" % my_stuff)
This code tested in Python 2.7.3 and 3.3.5
With python3 optional closefd argument is recognized.
If set to False, resulting IO object won't close underlying fd:
if file_path != None:
outputHandle = open(file_path, 'w')
else:
outputHandle = open(sys.stdout.fileno(), 'w', closefd=False)
with outputHandle as handle:
print(my_stuff, file=handle)