Develop Reference

How to specify file paths in a compressed way?

I'm in need fo specifying file paths to sort JSON files that gets retreived from an API. I have a class module that saves and loads the files
import os
import json
class Directory:
def __init__(self):
self.working_dir = os.path.dirname(__file__) #Get's the current working directory
def mkdir(self, *path):
"""Creates folder in the same level as the working directory folder
Args:
*args: path to folder that is to be created
"""
target_dir = os.path.join(self.working_dir, *path)
try:
if os.path.exists(target_dir) == True:
print(target_dir, 'exists')
else:
os.mkdir(os.path.join(self.working_dir, *path))
print(os.path.join(self.working_dir, *path), 'succesfully created')
except OSError as e:
print(folder,'coult not be created', e)
def check_if_file_exist(self, *path):
if os.path.exists(os.path.join(self.working_dir, *path)):
return True
else:
return False
def save_json(self, filename, content, *path):
"""save dictionarirys to .json files
Args:
file (str): The name of the file that is to be saved in .json format
filename (dict): The dictionary that is to be wrote to the .json file
folder (str): The folder name in the target directory
"""
target_dir = os.path.join(self.working_dir, *path)
file_dir = os.path.join(self.working_dir, target_dir, filename)
with open(file_dir + '.json', "w") as f:
#pretty prints and writes the same to the json file
f.write(json.dumps(content, indent=4, sort_keys=False))
But it often result in hideously long lines when a file-path has to be specified, for example.
#EXAMPLE
filename = self.league + '_' + year + '_' + 'fixturestats'
self.dir.save_json(filename, stats, '..', 'json', 'params', 'stats')
path = '/'.join(('..', 'json', 'params'))
#OTHER EXAMPLE
league_season_info = self.dir.load_json('season_params.json', '..', 'json', 'params')
I'm wondering what optimal praxis is when having a repository that has rather static folders relative to the working directory. All my modules are now build to create the folders that are needed in the repository if they do not exist, so I can count on that the paths exist when loading or saving files.

A bit late to the party, but here's how I might do it. As you mention these folders are fairly static, they can be placed into a configuration object of sorts (e.g. as class members of the thing owning dir, or a stand-alone object). Since I do not know your class structure, I'll opt for a standalone object.
This might look like the following, using pathlib:*
from pathlib import Path
class StorageConfig:
STORAGE_BASE_DIR = Path("json")
PARAMS_DIR = STORAGE_BASE_DIR / "params"
STATS_DIR = PARAMS_DIR / "stats"
# Etc.
This can be used as follows:
self.dir.save_json(filename, stats, StorageConfig.STATS_DIR)
params = self.dir.load_json('season_params.json', StorageConfig.PARAMS_DIR)
Or, doing away with the directory argument altogether:
self.dir.save_json(StorageConfig.STATS_DIR / (filename + ".json"), stats)
params = self.dir.load_json(StorageConfig.PARAMS_DIR / 'season_params.json')
*Pathlib can simplify most of the code currently in your Directory class. Have a look!

Reading a .json file in static folder

I'm trying to read a .json file from within my Flask application using:
def renderblog():
with open(url_for("static", filename="blogs.json")) as blog_file:
data = json.load(blog_file)
However I get the error:
FileNotFoundError: [Errno 2] No such file or directory: '/static/blogs.json'
Now I know for a fact that the directory exists within my project structure, but I have no idea why I'm getting this error. Any ideas? Is there a specific way to retrieve .json in Flask?

You generated a URL path, not a path to the local static folder. Use the app.static_folder attribute instead:
def renderblog():
filename = os.path.join(app.static_folder, 'blogs.json')
with open(filename) as blog_file:
data = json.load(blog_file)

Python open filename from custom PATH

Similar to the system path, I want to offer some convenience in my code allowing a user to specify a file name that could be in one of a handful of paths.
Say I had two or more config paths
['~/.foo-config/', '/usr/local/myapp/foo-config/']
And my user wants to open bar, (AKA bar.baz)
Is there a convenient build in way to let open('bar') or open('bar.baz') automatically search these paths for that file in LTR order of precedence? Eg, will temporary adjusting my sys.path to only be these directories do this for me?
Else, how would you suggest implementing a PATH-like searching open-wrapper?

As other people already mentioned: sys.path only affects the module search path, i.e. it's relevant for importing Python modules, but not at all for open().
I would suggest separating the logic for searching the paths in order of precedence and opening the file, because that way it's easier to test and read.
I would do something like this:
import os
PATHS = ['~/.foo-config/', '/usr/local/myapp/foo-config/']
def find_first(filename, paths):
for directory in paths:
full_path = os.path.join(directory, filename)
if os.path.isfile(full_path):
return full_path
def main():
filename = 'file.txt'
path = find_first(filename, PATHS)
if path:
with open(path) as f:
print f
else:
print "File {} not found in any of the directories".format(filename)
if __name__ == '__main__':
main()

open doesn't get into that kind of logic. If you want, write a wrapper function that uses os.path.join to join each member of sys.path to the parameter filename, and tries to open them in order, handling the error that occurs when no such file is found.
I'll add that, as another user stated, this is kind of a misuse of sys.path, but this function would work for any list of paths. Indeed, maybe the nicest option is to use the environment variables suggested by another user to specify a colon-delimited list of config directories, which you then parse and use within your search function.

environmental variables
say your app is named foo ... in the readme tell the user to use the FOO_PATH environmental variable to specify the extra paths
then inside your app do something like
for path in os.environ.get("FOO_PATH",".").split(";"):
lookfor(os.path.join(path,"somefile.txt"))
you could wrap it into a generic function
def open_foo(fname):
for path in os.environ.get("FOO_PATH",".").split(";"):
path_to_test = os.path.join(path,"somefile.txt")
if os.path.exists(path_to_test):
return open(path_to_test)
raise Exception("No File Found On FOOPATH")
then you could use it just like normal open
with open_foo("my_config.txt") as f:
print f.read()

Extract from Python Standard Library documentation for open built-in function:
open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None)
...file is either a string or bytes object giving the pathname (absolute or relative to the current working directory) of the file to be opened ...
Explicitely, open does not bring anything to automagically find a file : if path is not absolute, it is only searched in current directory.
So you will have to use a custom function or a custom class for that. For example:
class path_opener(object):
def __init__(self, path = [.]):
self.path = path
def set(self, path):
self.path = path
def append(self, path):
self.path.append(path)
def extent(self, path):
self.path.extend(path)
def find(self, file):
for folder in self.path:
path = os.path.join(folder, file)
if os.path.isfile(path):
return path
raise FileNotFoundError()
def open(self, file, *args, **kwargs):
return open(self.find(file), *args, **kwargs)
That means that a file opener will keep its own path, will be initialized by default with current path, will have methods to set, append to or extend its path, and will normaly raise a FileNotFoundError is a file is not found in any of the directories listed in its path.
Usage :
o = path_opener(['~/.foo-config/', '/usr/local/myapp/foo-config/'])
with o.open('foo') as fd:
...

Pyramid: How can I making a static view to some absolute path, and then let users upload files to that path?

In my view callable, I want users to be able to create a new file called filename like so:
#view_config(route_name='home_page', renderer='templates/edit.pt')
def home_page(request):
if 'form.submitted' in request.params:
name= request.params['name']
input_file=request.POST['stl'].filename
vertices, normals = [],[]
for line in input_file:
parts = line.split()
if parts[0] == 'vertex':
vertices.append(map(float, parts[1:4]))
elif parts[0] == 'facet':
normals.append(map(float, parts[2:5]))
ordering=[]
N=len(normals)
...parsing data...
data=[vertices,ordering]
jsdata=json.dumps(data)
renderer_dict = dict(name=name,data=jsdata)
app_dir = request.registry.settings['upload_dir']
filename = "%s/%s" % ( app_dir , name )
html_string = render('tutorial:templates/view.pt', renderer_dict, request=request)
with open(filename,'w') as file:
file.write(new_comment)
return HTTPFound(location=request.static_url('tutorial:pages/%(pagename)s.html' % {'pagename': name}))
return {}
right now, when I attempt to upload a file, I am getting this error message: IOError: [Errno 2] No such file or directory: u'/path/pages/one' (one is the name variable) I believe this is because I am incorrectly defining the app_dir variable. I want filename to be the url of the new file that is being created with the name variable that is defined above (so that it can be accessed at www.domain.com/pages/name). Here is the file structure of my app:
env
tutorial
tutorial
templates
home.pt
static
pages
(name1)
(name2)
(name3)
....
views.py
__init__.py
In my init.py I have:
config.add_static_view(name='path/pages/', path=config.registry.settings['upload_dir'])
In my development.ini file I have
[app:main]
use = egg:tutorial
upload_dir = /path/pages
Edit: If anyone has an idea on why this question isn't getting much attention, I would love to hear it.

While I feel like you probably have a misunderstanding of how to serve up user-generated content, I will show you a way to do what you're asking. Generally user-generated content would not be uploaded into your source, you'll provide some configurable spot outside to place it, as I show below.
Make the path configurable via your INI file:
[app:main]
use = egg:tutorial
upload_dir = /path/to/writable/upload/directory
Add a static view that can serve up files under that directory.
config.add_static_view(name='/url/to/user_uploads', path=config.registry.settings['upload_dir'])
In your upload view you can get your app_dir via
app_dir = request.registry.settings['upload_dir']
Copy the data there, and from then on it'll be available at /url/to/user_uploads/filename.

Django unzip a file, add contents to database

I am trying to create a system that enables the admin to upload a zipfile, then the script will automatically, using signals, unzip it, search for all the files in jpg,png. create a list of them and generate a database record according to it.
In models, i have Project and Photo table, Photo has Many-to-One aka Foreign Key relationship with Project.
The script below is with the signal i am working. I can get instance.file_zip.path without errors, and the script works well when run manually.
Long-time debugging and I assume that there is something wrong with belongs_to=instance but I do not know how to fix it as I didn't actually understand why it gaves an error.
The extraction part works fine, I just put them here for reference, most likely you will not need to read and understand it.
#receiver(post_save, sender=Project)
def unzip_and_process(sender, instance, **kwargs):
#project_zip = FieldFile.open(file_zip, mode='rb')
file_path = instance.file_zip.path
file_list = []
with zipfile.ZipFile(file_path, 'r') as project_zip:
project_zip.extractall(re.search('[^\s]+(?=\.zip)', file_path).group(0))
project_zip.close()
for root, dirs, files in os.walk(file_path):
for filename in files:
file_list.append(os.path.join(root, filename))
photo_list = filter(filter_photos, file_list)
for photo in photo_list:
print 'Processing %s'%photo
p = Photo.objects.create(belongs_to=instance, img=photo, desc='Processed from zipfile')
p.save()
update
class Project(models.Model):
project_name=models.CharField(max_length=150)
date_taken=models.DateField()
date_deadline=models.DateField()
price=models.FloatField()
price_paid=models.BooleanField()
owner=models.ForeignKey(User)
file_zip=models.FileField(upload_to='projects/%Y/%m/%d')
def __unicode__(self):
return self.project_name
def file_path(self):
return re.search('[^\s]+(?=\.zip)', self.file_zip.name).group(0)
class Photo(models.Model):
def project_path(instance, filename):
return '%s/%s'%(instance.belongs_to.file_path(),filename)
belongs_to=models.ForeignKey(Project, verbose_name="related_project")
img=models.ImageField(upload_to=project_path, max_length=255)
desc=models.CharField(max_length=255)
def __unicode__(self):
return '%s FROM [%s]'%(self.img.name,self.belongs_to)

django-photologue has something extractly what you want, and they created a similiar hack to upload zipfile.
Link: http://code.google.com/p/django-photologue/ incase you don't want to google
Evenmore, the zip uploading class is GalleryUpload(models.Model)

for root, dirs, files in os.walk(file_path):
file_path refers to a zip file. not a directory hence os.walk returns nothing