I have a function that loads data from a data.json file defined in models.py as follow:
def load_data():
file_path = Path(__file__).parent / 'data.json'
with open(file_path, 'r') as file:
data = json.load(file)['data']
return data
loaded_data = load_data()
I used loaded_data throughout all fuctions defined in models.py. The data.json file contains a JSON array.
My test_models.py is as follow:
from unittest.mock import patch
from models import ... (a list of function to test)
# For replaceing model.load_data()
mock_data = []
def get_mock_data():
return mock_data
#patch('models.load_data', side_effect= get_mock_restaurants)
class TestRestaurantsModel(unittest.TestCase):
However, somehow the real models.load_data still get executed. I know it because I changed the file_path to randomabc.json and got FileNotFoundError. How to I prevent the execution of models.load_data? I do not need to mock models.load_data essentially. I just need to prevent its execution during the test and assigned a mock data to models.data.
Related
I'm trying to test this code:
def read_classes(file):
if CLASSES in file:
classes = open(file, "rb").read()
else:
with ZipFile(file, "r") as archive:
classes = archive.read(CLASSES)
return classes
What is important for me is, when the provided file contains CLASSES in its name, open will be called, otherwise, ZipFile will be used. The first part I was able to test already, however, I cannot mock ZipFile in order to return a mocked object (archive) - which I then can assert that had the read method called. This is what I've been trying so far:
#patch('zipfile.ZipFile')
def test_givenFile_whenReadClasses_expectArchiveCalled(self, mock_zipfile):
file = 'sample.file'
archive = Mock()
mock_zipfile.return_value = archive
read_classes(file)
archive.read.assert_called_once_with("classes.file")
When I do that, it continues to execute the original ZipFile constructor, giving me:
IOError: [Errno 2] No such file or directory: 'sample.file'
Straight to the point:
#patch('zipfile.ZipFile')
def test_givenFile_whenReadClasses_expectArchiveCalled(self, mocked_zip_file):
file = 'file'
archive = Mock()
mocked_read = Mock()
archive.return_value.read = mocked_read
mocked_zip_file.return_value.__enter__ = archive
read_classes(dex_file)
mocked_read.assert_called_once_with('another_file')
I'm using pytest and want to test that a function writes some content to a file. So I have writer.py which includes:
MY_DIR = '/my/path/'
def my_function():
with open('{}myfile.txt'.format(MY_DIR), 'w+') as file:
file.write('Hello')
file.close()
I want to test /my/path/myfile.txt is created and has the correct content:
import writer
class TestFile(object):
def setup_method(self, tmpdir):
self.orig_my_dir = writer.MY_DIR
writer.MY_DIR = tmpdir
def teardown_method(self):
writer.MY_DIR = self.orig_my_dir
def test_my_function(self):
writer.my_function()
# Test the file is created and contains 'Hello'
But I'm stuck with how to do this. Everything I try, such as something like:
import os
assert os.path.isfile('{}myfile.txt'.format(writer.MYDIR))
Generates errors which lead me to suspect I'm not understanding or using tmpdir correctly.
How should I test this? (If the rest of how I'm using pytest is also awful, feel free to tell me that too!)
I've got a test to work by altering the function I'm testing so that it accepts a path to write to. This makes it easier to test. So writer.py is:
MY_DIR = '/my/path/'
def my_function(my_path):
# This currently assumes the path to the file exists.
with open(my_path, 'w+') as file:
file.write('Hello')
my_function(my_path='{}myfile.txt'.format(MY_DIR))
And the test:
import writer
class TestFile(object):
def test_my_function(self, tmpdir):
test_path = tmpdir.join('/a/path/testfile.txt')
writer.my_function(my_path=test_path)
assert test_path.read() == 'Hello'
Currently, I have a file called utils.py where I keep all my functions and another file called main.py.
In my utils file, I have a two functions that load and save to a json file, along with a bunch of other functions that will edit the data.
def save_league(league_name, records):
with open('%s.json' % league_name, 'w') as f:
f.write(json.dumps(records))
def load_league(league_name):
with open('%s.json' % league_name, 'r') as f:
content = f.read()
records = json.loads(content)
return records
I am trying to add optional arguments for the save_league function by changing the function to:
def save_league(name = league_name, r = records):
with open('%s.json' % name, 'w') as f:
f.write(json.dumps(r))
This way the file will save just from save_league().
However, when I try to import a function with optional arguments in main.py, I get a name error because the default arguments are not set at the beginning.
NameError: name 'league_name' is not defined
Is it possible import functions with optional args into another file or do I have to combine the two files into one?
I'm trying to run some analysis of class and function dependencies in a Python code base. My first step was to create a .csv file for import into Excel using Python's csv module and regular expressions.
The current version of what I have looks like this:
import re
import os
import csv
from os.path import join
class ClassParser(object):
class_expr = re.compile(r'class (.+?)(?:\((.+?)\))?:')
python_file_expr = re.compile(r'^\w+[.]py$')
def findAllClasses(self, python_file):
""" Read in a python file and return all the class names
"""
with open(python_file) as infile:
everything = infile.read()
class_names = ClassParser.class_expr.findall(everything)
return class_names
def findAllPythonFiles(self, directory):
""" Find all the python files starting from a top level directory
"""
python_files = []
for root, dirs, files in os.walk(directory):
for file in files:
if ClassParser.python_file_expr.match(file):
python_files.append(join(root,file))
return python_files
def parse(self, directory, output_directory="classes.csv"):
""" Parse the directory and spit out a csv file
"""
with open(output_directory,'w') as csv_file:
writer = csv.writer(csv_file)
python_files = self.findAllPythonFiles(directory)
for file in python_files:
classes = self.findAllClasses(file)
for classname in classes:
writer.writerow([classname[0], classname[1], file])
if __name__=="__main__":
parser = ClassParser()
parser.parse("/path/to/my/project/main/directory")
This generates a .csv output in format:
class name, inherited classes (also comma separated), file
class name, inherited classes (also comma separated), file
... etc. ...
I'm at the point where I'd like to start parsing function declaration and definitions in addition to the class names. My question: Is there a better way to get the class names, inherited class names, function names, parameter names, etc.?
NOTE: I've considered using the Python ast module, but I don't have experience with it and don't know how to use it to get the desired information or if it can even do that.
EDIT: In response to Martin Thurau's request for more information - The reason I'm trying to solve this issue is because I've inherited a rather lengthy (100k+ lines) project that has no discernible structure to its modules, classes and functions; they all exist as a collection of files in a single source directory.
Some of the source files contain dozens of tangentially related classes and are 10k+ lines long which makes them difficult to maintain. I'm starting to perform analysis for the relative difficulty of taking every class and packaging it into a more cohesive structure using The Hitchhiker's Guide to Packaging as a base. Part of what I care about for that analysis is how intertwined a class is with other classes in its file and what imported or inherited classes a particular class relies on.
I've made a start on implementing this. Put the following code in a file, and run it, passing the name of a file or directory to analyse. It will print out all the classes it finds, the file it was found in, and the bases of the class. It is not intelligent, so if you have two Foo classes defined in your code base it will not tell you which one is being used, but it is a start.
This code uses the python ast module to examine .py files, and finds all the ClassDef nodes. It then uses this meta package to print bits of them out - you will need to install this package.
$ pip install -e git+https://github.com/srossross/Meta.git#egg=meta
Example output, run against django-featured-item:
$ python class-finder.py /path/to/django-featured-item/featureditem/
FeaturedField,../django-featured-item/featureditem/fields.py,models.BooleanField
SingleFeature,../django-featured-item/featureditem/tests.py,models.Model
MultipleFeature,../django-featured-item/featureditem/tests.py,models.Model
Author,../django-featured-item/featureditem/tests.py,models.Model
Book,../django-featured-item/featureditem/tests.py,models.Model
FeaturedField,../django-featured-item/featureditem/tests.py,TestCase
The code:
# class-finder.py
import ast
import csv
import meta
import os
import sys
def find_classes(node, in_file):
if isinstance(node, ast.ClassDef):
yield (node, in_file)
if hasattr(node, 'body'):
for child in node.body:
# `yield from find_classes(child)` in Python 3.x
for x in find_classes(child, in_file): yield x
def print_classes(classes, out):
writer = csv.writer(out)
for cls, in_file in classes:
writer.writerow([cls.name, in_file] +
[meta.asttools.dump_python_source(base).strip()
for base in cls.bases])
def process_file(file_path):
root = ast.parse(open(file_path, 'r').read(), file_path)
for cls in find_classes(root, file_path):
yield cls
def process_directory(dir_path):
for entry in os.listdir(dir_path):
for cls in process_file_or_directory(os.path.join(dir_path, entry)):
yield cls
def process_file_or_directory(file_or_directory):
if os.path.isdir(file_or_directory):
return process_directory(file_or_directory)
elif file_or_directory.endswith('.py'):
return process_file(file_or_directory)
else:
return []
if __name__ == '__main__':
classes = process_file_or_directory(sys.argv[1])
print_classes(classes, sys.stdout)
I am wondering whether there is a way to upload a zip file to django web server and put the zip's files into django database WITHOUT accessing the actual file system in the process (e.g. extracting the files in the zip into a tmp dir and then load them)
Django provides a function to convert python File to Django File, so if there is a way to convert ZipExtFile to python File, it should be fine.
thanks for help!
Django model:
from django.db import models
class Foo:
file = models.FileField(upload_to='somewhere')
Usage:
from zipfile import ZipFile
from django.core.exceptions import ValidationError
from django.core.files import File
from io import BytesIO
z = ZipFile('zipFile')
istream = z.open('subfile')
ostream = BytesIO(istream.read())
tmp = Foo(file=File(ostream))
try:
tmp.full_clean()
except Validation, e:
print e
Output:
{'file': [u'This field cannot be blank.']}
[SOLUTION] Solution using an ugly hack:
As correctly pointed out by Don Quest, file-like classes such as StringIO or BytesIO should represent the data as a virtual file. However, Django File's constructor only accepts the build-in file type and nothing else, although the file-like classes would have done the job as well. The hack is to set the variables in Django::File manually:
buf = bytesarray(OPENED_ZIP_OBJECT.read(FILE_NAME))
tmp_file = BytesIO(buf)
dummy_file = File(tmp_file) # this line actually fails
dummy_file.name = SOME_RANDOM_NAME
dummy_file.size = len(buf)
dummy_file.file = tmp_file
# dummy file is now valid
Please keep commenting if you have a better solution (except for custom storage)
There's an easier way to do this:
from django.core.files.base import ContentFile
uploaded_zip = zipfile.ZipFile(uploaded_file, 'r') # ZipFile
for filename in uploaded_zip.namelist():
with uploaded_zip.open(filename) as f: # ZipExtFile
my_django_file = ContentFile(f.read())
Using this, you can convert a file that was uploaded to memory directly to a django file. For a more complete example, let's say you wanted to upload a series of image files inside of a zip to the file system:
# some_app/models.py
class Photo(models.Model):
image = models.ImageField(upload_to='some/upload/path')
...
# Upload code
from some_app.models import Photo
for filename in uploaded_zip.namelist():
with uploaded_zip.open(filename) as f: # ZipExtFile
new_photo = Photo()
new_photo.image.save(filename, ContentFile(f.read(), save=True)
Without knowing to much about Django, i can tell you to take a look at the "io" package.
You could do something like:
from zipfile import ZipFile
from io import StringIO
zname,zipextfile = 'zipcontainer.zip', 'file_in_archive'
istream = ZipFile(zname).open(zipextfile)
ostream = StringIO(istream.read())
And then do whatever you would like to do with your "virtual" ostream Stream/File.
I've used the following django file class to avoid the need to read ZipExtFile into a another datastructure (StingIO or BytesIO) while properly impelementing what Django needs in order to save the file directly.
from django.core.files.base import File
class DjangoZipExtFile(File):
def __init__(self, zipextfile, zipinfo):
self.file = zipextfile
self.zipinfo = zipinfo
self.mode = 'r'
self.name = zipinfo.filename
self._size = zipinfo.file_size
def seek(self, position):
if position != 0:
#this will raise an unsupported operation
return self.file.seek(position)
#TODO if we have already done a read, reopen file
zipextfile = archive.open(path, 'r')
zipinfo = archive.getinfo(path)
djangofile = DjangoZipExtFile(zipextfile, zipinfo)
storage = DefaultStorage()
result = storage.save(djangofile.name, djangofile)