Develop Reference

Is there an easy way to automate keeping track of files generated and saved in python?

I don't know if there is an easy way of doing this that doesn't rely on manually writing down what the saved outputs from a script are so open to any suggestions.
I want a function that runs at the end of my script and that automatically generates a text file with a name like:
"IO_track_scriptname_date_time"
Which has a list of the files I loaded and the files I saved (location links).
And then saves this txt file to the desired destination.
Thank you for your help
Edit:
Or any alternative way of keeping a log of inputs and outputs.

Here is a thin object wrapper around the open function that tracks all of the files that are opened.
class Open:
_open = open
def __init__(self):
self.opened_files = []
self.fp = None
def __call__(self,
file,
mode='r',
buffering=-1,
encoding=None,
errors=None,
newline=None,
closefd=True,
opener=None):
self.fp = self._open(file, mode, buffering, encoding, errors,
newline, closefd, opener)
self.opened_files.append((mode, file))
return self.fp
def __enter__(self, *args, **kwargs):
return self.__call__(*args, **kwargs)
def __exit__(self, *exc_details):
return self.fp.close()
def __getattr__(self, attr):
return getattr(self.fp, attr)
def export(self, filename):
with open(filename, 'w') as fp:
for m, fn in self.opened_files:
fp.write(f'({m}): {fn}\n')
To actually use it, you will need to overwrite the built-in open function with an instantiation of this class. If you have one file that you are calling, you can pop this into the __main__ block. i.e.
...
if __name__=='__main__':
# code defining Open class here
...
open = Open()
# other code in __main__ here
open.export("IO_track_scriptname_date_time.txt")

How can I mock ZipFile constructor?

I'm trying to test this code:
def read_classes(file):
if CLASSES in file:
classes = open(file, "rb").read()
else:
with ZipFile(file, "r") as archive:
classes = archive.read(CLASSES)
return classes
What is important for me is, when the provided file contains CLASSES in its name, open will be called, otherwise, ZipFile will be used. The first part I was able to test already, however, I cannot mock ZipFile in order to return a mocked object (archive) - which I then can assert that had the read method called. This is what I've been trying so far:
#patch('zipfile.ZipFile')
def test_givenFile_whenReadClasses_expectArchiveCalled(self, mock_zipfile):
file = 'sample.file'
archive = Mock()
mock_zipfile.return_value = archive
read_classes(file)
archive.read.assert_called_once_with("classes.file")
When I do that, it continues to execute the original ZipFile constructor, giving me:
IOError: [Errno 2] No such file or directory: 'sample.file'

Straight to the point:
#patch('zipfile.ZipFile')
def test_givenFile_whenReadClasses_expectArchiveCalled(self, mocked_zip_file):
file = 'file'
archive = Mock()
mocked_read = Mock()
archive.return_value.read = mocked_read
mocked_zip_file.return_value.__enter__ = archive
read_classes(dex_file)
mocked_read.assert_called_once_with('another_file')

Why isn't context manager closing file descriptor?

I'm trying to create a context manager that uses mmap which is itself is a context manager. Initially I had an dumb open file problem Why isn't mmap closing associated file (getting PermissionError: [WinError 32])? and an answer quickly explained why it wasn't working as desired.
Given that information, I've attempted two different ways to correct the issue, but neither one has worked.
The first approach was to use thecontextlib's #contextmanager decorator:
from contextlib import contextmanager
import os
import mmap
#contextmanager
def memory_map(filename, access=mmap.ACCESS_WRITE):
size = os.path.getsize(filename)
fd = os.open(filename, os.O_RDWR)
print('about to yield')
with mmap.mmap(fd, size, access=access) as m:
yield m
print('in finally clause')
os.close(fd) # Close the associated file descriptor.
test_filename = 'data'
# First create the test file.
size = 1000000
with open(test_filename, 'wb') as f:
f.seek(size - 1)
f.write(b'\x00')
# Read and modify mmapped file in-place.
with memory_map(test_filename) as m: # Causes AttributeError: __enter__
print(len(m))
print(m[0:10])
# Reassign a slice.
m[0:11] = b'Hello World'
# Verify that changes were made
print('reading back')
with open(test_filename, 'rb') as f:
print(f.read(11))
# Delete test file.
# Causes:
# PermissionError: [WinError 32] The process cannot access the file because it
# is being used by another process: 'data'
os.remove(test_filename)
But it results in:
Traceback (most recent call last):
File "memory_map.py", line 27, in <module>
with memory_map(test_filename) as m: # Causes AttributeError: __enter__
AttributeError: __enter__
In the next attempt I tried explicitly creating a context manager class:
import os
import mmap
class MemoryMap:
def __init__(self, filename, access=mmap.ACCESS_WRITE):
print('in MemoryMap.__init__')
size = os.path.getsize(filename)
self.fd = os.open(filename, os.O_RDWR)
self.mmap = mmap.mmap(self.fd, size, access=access)
def __enter__(self):
print('in MemoryMap.__enter__')
return self.mmap
def __exit__(self, exc_type, exc_value, traceback):
print('in MemoryMap.__exit__')
os.close(self.fd) # Close the associated file descriptor.
print(' file descriptor closed')
test_filename = 'data'
# First create the test file.
size = 1000000
with open(test_filename, 'wb') as f:
f.seek(size - 1)
f.write(b'\x00')
# Read and modify mmapped file in-place.
with MemoryMap(test_filename) as m:
print(len(m))
print(m[0:10])
# Reassign a slice.
m[0:11] = b'Hello World'
# Verify that changes were made
print('reading back')
with open(test_filename, 'rb') as f:
print(f.read(11))
# Delete test file.
# Causes PermissionError: [WinError 32] The process cannot access the file
# because it is being used by another process: 'data'
os.remove(test_filename)
This makes it further, but the PermissionError is back—which really confuses me because the file descriptor was closed in that version as you can see in the output produced:
in MemoryMap.__init__
in MemoryMap.__enter__
1000000
b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
in MemoryMap.__exit__
file descriptor closed
reading back
b'Hello World'
Traceback (most recent call last):
File "memory_map2.py", line 47, in <module>
os.remove(test_filename)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process: 'data'
So it seems I'm stuck again. Any ideas on what's wrong (as well as how to fix it)? Also, in the event they can both be fixed, which one is better if you have an opinion?
Solutions
There was an error in both snippets. This first was a simple typographical error. The contextmanger decorator was commented out. Should have been:
#contextmanager # Leading "#" changed to "#".
def memory_map(filename, access=mmap.ACCESS_WRITE):
size = os.path.getsize(filename)
fd = os.open(filename, os.O_RDWR)
...
In the second it was because the mmap itself was not being closed in the __exit__() method, just the associated file descriptor. That never occurred to me because the exception raised was the same as in the first case.
def __exit__(self, exc_type, exc_value, traceback):
print('in MemoryMap.__exit__')
self.mmap.close() # ADDED.
os.close(self.fd) # Close the associated file descriptor.
print(' file descriptor closed')

In case of your second attempt, you need to close the memory mapped file:
def __exit__(self, exc_type, exc_value, traceback):
self.mm.close()
print('in MemoryMap.__exit__')
os.close(self.fd) # Close the associated file descriptor.
print(' file descriptor closed')

Using tempfile to create pdf/xls documents in flask

I wanted to ask if it's possible to create PDF/XLS documents as temporary files. I'm doing that to send them using flask afterwards. For pdf/xls files creation I use reportlab and xlsxwriter packages respectively. When I save document using their methods, I get the "Python temporary file permission denied" error. When I try to close using the tempfile methods, files become corrupted. Is there any way to overcome this? Or any other suitable solution?
EDIT:
Some code snippets:
import xlswriter
import tempfile
from flask import after_this_request
#app.route('/some_url', method=['POST'])
def create_doc_function():
#after_this_request
def cleanup(response):
temp.close()
return response
temp = tempfile.TemporaryFile()
book = xlsxwriter.Workbook(temp.name)
# some actions here ...
book.close() # raises "Python temporaty file permission denied" error.
# If missed, Excel book is gonna be corrupted,
# i.e. blank, which make sense
return send_file(temp, as_attachment=True,
attachment_filename='my_document_name.xls')
Similar story with pdf files.

Use tempfile.mkstemp() which will create a standard temp file on disk which will persist until removed:
import tempfile
import os
handle, filepath = tempfile.mkstemp()
f = os.fdopen(handle) # convert raw handle to file object
...
EDIT
tempfile.TemporaryFile() will be destroyed as soon as it's closed, which is why your code above is failing.

You can use and delete NamedTemporaryFile with context manager (or atexit module). It may do the dirty job for you.Example 1:
import os
from tempfile import NamedTemporaryFile
# define class, because everyone loves objects
class FileHandler():
def __init__(self):
'''
Let's create temporary file in constructor
Notice that there is no param (delete=True is not necessary)
'''
self.file = NamedTemporaryFile()
# write something funny into file...or do whatever you need
def write_into(self, btext):
self.file.write(btext)
def __enter__(self):
'''
Define simple but mandatory __enter__ function - context manager will require it.
Just return the instance, nothing more is requested.
'''
return self
def __exit__(self, exc_type, exc_val, exc_tb):
'''
Also define mandatory __exit__ method which is called at the end.
NamedTemporaryFile is deleted as soon as is closed (function checks it before and after close())
'''
print('Calling __exit__:')
print(f'File exists = {os.path.exists(self.file.name)}')
self.file.close()
print(f'File exists = {os.path.exists(self.file.name)}')
# use context mamager 'with' to create new instance and do something
with FileHandler() as fh:
fh.write_into(b'Hi happy developer!')
print(f'\nIn this point {fh.file.name} does not exist (exists = {os.path.exists(fh.file.name)})')
Output:
Calling __exit__:
File exists = True
File exists = False
In this point D:\users\fll2cj\AppData\Local\Temp\tmpyv37sp58 does not exist (exists = False)
Or you can use atexit module which calls defined function when program (cmd) exits.Example 2:
import os, atexit
from tempfile import NamedTemporaryFile
class FileHandler():
def __init__(self):
self.file = NamedTemporaryFile()
# register function called when quit
atexit.register(self._cleanup)
def write_into(self, btext):
self.file.write(btext)
def _cleanup(self):
# because self.file has been created without delete=False, closing the file causes its deletion
self.file.close()
# create new instance and do whatever you need
fh = FileHandler()
fh.write_into(b'Hi happy developer!')
# now the file still exists, but when program quits, _cleanup() is called and file closed and automaticaly deleted.

Get file objects inside a scope of "with" block

For example there's with statement:
with open("ACCELEROMETER", 'w') as ACCELEROMETER,\
open('GPS', 'w') as GPS,\
open('LIGHT', 'w') as LIGHT,\
open('LOCATION', 'w') as LOCATION,\
open('MIC', 'w') as MIC,\
open('SCREEN', 'w') as SCREEN,\
open('TIME', 'w') as TIME:
I want to get file objects just created using some python code :
I'm looking for equivalent of dir function for local scope of with.
Is it possible?

What you're asking for isn't really possible (without making some assumptions) since with doesn't create a new namespace. You could create a file-list object which is implemented as a context manager ...
class FileList(list):
def __init__(self, files, mode='r'):
list.__init__(open(arg, mode) for arg in files)
def __enter__(self):
return self
def __exit__(self, *args):
for fobj in self:
fobj.close()
with FileList(["ACCELEROMETER", "GPS", ...], mode='w') as fl:
for fobj in fl:
...