Develop Reference

Python - too many values to unpack while looking for a file

I'm trying to get a message that file not found when it doesn't match the file_name*.txt pattern in specific directory.
When calling the script with file_name*.txt argument, all works fine. While entering invalid name file_*.txt throws:
File "etl.py", line 14, in main
path, file = file_path.get_source_file_path()
ValueError: too many values to unpack (expected 2)
Why is this happening?
import fnmatch
import os
class FilePrep:
def __init__(self, path_dir, file_name):
self.path = path_dir
self.file_name = file_name
def get_source_file_path(self):
source_file = []
for file_name in os.listdir(self.path):
if fnmatch.fnmatch(file_name, self.file_name):
source_file.append(file_name)
try:
source_file_name = str(source_file[0])
except IndexError:
return "file not found"
file_path = os.path.join(self.path, source_file_name)
return file_path, source_file_name
main.py
import file_prep
import xml.etree.ElementTree as element_tree
import pandas
import sys
def main():
dir = sys.argv[1]
file_input = sys.argv[2]
#python3 etl.py /home/user/projects/python/folder/ file_name*.xml
file_path = file_prep.FilePrep(dir, file_input)
path, file = file_path.get_source_file_path()
print(path)

Your problem is in this line in main.py:
path, file = file_path.get_source_file_path()
here you are unpacking the return value of get_source_file_path() into two variables. This works fine if the file exists (because you actually return two values) but does not when the file does not exist since you only return one value return "file not found".
To fix this I would raise an error instead of returning a string message in case of failure. Your code can become:
import fnmatch
import os
class FileNotFoundError(Exception):
pass
class FilePrep:
def __init__(self, path_dir, file_name):
self.path = path_dir
self.file_name = file_name
def get_source_file_path(self):
source_file = []
for file_name in os.listdir(self.path):
if fnmatch.fnmatch(file_name, self.file_name):
source_file.append(file_name)
try:
source_file_name = str(source_file[0])
except IndexError:
raise FileNotFoundError(f"file {self.file_name} not found")
file_path = os.path.join(self.path, source_file_name)
return file_path, source_file_name
import file_prep
import xml.etree.ElementTree as element_tree
import pandas
import sys
def main():
dir = sys.argv[1]
file_input = sys.argv[2]
#python3 etl.py /home/user/projects/python/folder/ file_name*.xml
file_path = file_prep.FilePrep(dir, file_input)
try:
path, file = file_path.get_source_file_path()
print(path)
except file_prep.FileNotFoundError as e:
print(e)

The previous answers somewhat got it the wrong way, saying in case of failure, "only one" value is returned. While that's true, it doesn't explain why the error message says there are "too many values to unpack", since 1 is not more than 2. The reason is that that one value is a string, which, due to to the multi-assignment, will be treated as an iterable. Since it has 14 characters, that's 14 values, and that's too many.

The issue is this following line:
except IndexError:
return "file not found"
When you hit that except condition your return from your function call returns only one variable, that string. You could modify your condition to do the following:
except IndexError:
source_file_name = "file not found"
However this won't work as you try to use this file name later on, a better approach would be to raise an error properly as shown in this answer by Matteo here.

When you call the function here, it's expected that two values will be returned
path, file = file_path.get_source_file_path()
When an exception occurs here, only one value is being returned which is causing the error
try:
source_file_name = str(source_file[0])
except IndexError:
return "file not found"
There are two ways to fix this.
Make it so only one value is expected to be returned and you modify the return at the bottom to only pass one value
You modify the exception so it returns two values

How to store declared variables in a array

I am currently working on a file sorting program and now that I have the selection of the Folders finished I need to assign the right paths to the variables to use them later on to move the files. My code right now looks like this
import os
import promptlib
sourcepath = str()
destpath = str()
pathimg = str()
pathtxt = str()
pathaudio = str()
pathvideo = str()
pathexe = str()
pathzips = str()
def get_paths():
global sourcepath
global destpath
sourcepath = promptlib.Files().dir()+"\\"
destpath = promptlib.Files().dir()+"\\"
def check_or_create_folder():
get_paths()
listToCreate = ["images\\","text documents\\","audio files\\","video files\\","executables\\","zips\\"]
pathToCreate = destpath+"sortedDownloads\\"
global paths
try: os.mkdir(pathToCreate)
except OSError: return
for elememt in listToCreate:
try: os.mkdir(pathToCreate+elememt)
except OSError: break
I thought about storing them in an array and then declaring them element by element maybe something like
def check_or_create_folder():
paths = [global pathimg, global pathtxt]
listToCreate = ["images\\","text documents\\","audio files\\","video files\\","executables\\","zips\\"]
pathToCreate = destpath+"sortedDownloads\\"
for i in range(len(listToCreate)):
try: os.mkdir(pathToCreate+listToCreate[i])
paths[i] = pathToCreate+listToCreate[i]
except OSError: break
but that doesn't work, I tried searching it up but I couldn't find anything so what would be your approach to this problem?
thank you before for answering.
(i am sorry for my English I'm still 15 years old and it's not that good)

Don't use global variables. Instead of using string operations to combine pathes, use pathlib.Path.
To store your created folders, just use a list:
import promptlib
from pathlib import Path
SUB_FOLDERS = ["images", "text documents", "audio files", "video files", "executables", "zips"]
def get_paths():
sourcepath = Path(promptlib.Files().dir())
destpath = Path(promptlib.Files().dir())
return sourcepath, destpath
def check_or_create_folder():
sourcepath, destpath = get_paths()
folders = []
for folder in SUB_FOLDERS:
folder = destpath / "sortedDownloads" / folder
folder.mkdir(parents=True, exist_ok=True)
folders.append(folder)
return sourcepath, folders

How can I pass key-value to decorator in python?

I am trying to validate the dictionary parameter.
import logging
import os
# decorator
def file_validator(f):
def wrapped(*args):
"""
Once there is passed values,
file_path = os.path.join(path_info, file_info)
if os.path.exists(file_path):
logging.info('{} exists'.format(file_info))
else:
logging.info('{} does not exist'.format(file_info))
"""
# original function
#file_validator
def original_function(file_dict):
# pass only specific element to file_validator decorator for checking
# for example only "pathA": "/files", "fileA": "bar.csv"
sample_dict = {"pathA": "/files", "fileA": "bar.csv", "fileB": "hello.txt"}
original_function(sample_dict)
Is there a way to check this way using decorator?
EDIT
This could be equivalent to what I want to do.
def file_validator(filepath, filename):
file_path = os.path.join(filepath + filename)
if os.path.exists(file_path):
logging.info('{} exists'.format(filename))
else:
logging.info('{} does not exist'.format(filename))
def original_function(file_dict):
file_validator(file_dict['pathA'], file_dict['fileA'])
file_validator(file_dict['pathA'], file_dict['fileB'])
sample_dict = {"pathA": "/files", "fileA": "bar.csv", "fileB": "hello.txt"}
original_function(sample_dict)

Seems like something like this should do the trick:
import os
import logging
def file_validator(func):
def wrapper(file_dict:dict):
# Turn file_dict to two lists:
# paths = ["/files"]
# files = ["bar.csv", "hello.txt"]
paths = [
path
for name, path in file_dict.items()
if name.startswith("path")
]
files = [
file
for name, file in file_dict.items()
if name.startswith("file")
]
# Loop through all the path & file combinations and check if they exist
for path in paths:
for file in files:
full_path = os.path.join(path, file)
if os.path.exists(full_path):
logging.info('{} exists'.format(file))
else:
logging.info('{} does not exist'.format(file))
# Run the actual function
return func(file_dict)
return wrapper
#file_validator
def original_function(file_dict):
...
files = {"pathA": "/files", "fileA": "bar.csv", "fileB": "hello.txt"}
original_function(files)
# Note that fileB is not checked as it's missing "pathB"
# but from the question it is unclear how this should be handled
But there is some code smell. If possible, I would advise not to store your paths and files in that way as that's not easy to manipulate and prone to bugs. Much better would be to store them as a list of full paths by creating all the combinations using itertools.combinations, or then just having two lists: paths and files.

logging.info replaced by print here for verification.
import logging
import os
def file_validator(f):
def wrapper(args):
for path, file in args.items():
file_path = os.path.join(path, file)
if os.path.exists(file_path):
print('{} exists'.format(file_path))
else:
print('{} does not exist'.format(file_path))
f(args)
return wrapper
#file_validator
def original_function(file_dict):
print(file_dict)
sample_dict = {"pathA": "\\files", "fileA": "bar.csv", "fileB": "hello.txt"}
original_function(sample_dict)

How to zip a folder in python with password?

With pyminizip i am able to zip a file with password in python :
filepath=r"C:\Users\xxx\Desktop\myFolder\file.txt"
import pyminizip
pyminizip.compress(filepath, None,"output.zip", "password", 0)
But how do I zip the whole folder 'myFolder' into a zip file with password?
I tried removing the filename from the path but it gives the error
OSError: error in opening C:\Users\xxx\Desktop\myFolder for reading
EDIT :
The below link has a function which will zip the directory. But It wont add a password.
https://www.calazan.com/how-to-zip-an-entire-directory-with-python/
If anyone can let me know if it is possible to add a password to an existing zip file, that will solve my problem. Is that possible?

I was finally able to accomplish encryping the whole directory(including all subfolder struncture and files) using a library called 'pyzipper' suggested by Anupam Chaplot.
Here is the solution :
def zip_folderPyzipper(folder_path, output_path):
"""Zip the contents of an entire folder (with that folder included
in the archive). Empty subfolders will be included in the archive
as well.
"""
parent_folder = os.path.dirname(folder_path)
# Retrieve the paths of the folder contents.
contents = os.walk(folder_path)
try:
zip_file = pyzipper.AESZipFile('new_test.zip','w',compression=pyzipper.ZIP_DEFLATED,encryption=pyzipper.WZ_AES)
zip_file.pwd=b'PASSWORD'
for root, folders, files in contents:
# Include all subfolders, including empty ones.
for folder_name in folders:
absolute_path = os.path.join(root, folder_name)
relative_path = absolute_path.replace(parent_folder + '\\',
'')
print ("Adding '%s' to archive." % absolute_path)
zip_file.write(absolute_path, relative_path)
for file_name in files:
absolute_path = os.path.join(root, file_name)
relative_path = absolute_path.replace(parent_folder + '\\',
'')
print ("Adding '%s' to archive." % absolute_path)
zip_file.write(absolute_path, relative_path)
print ("'%s' created successfully." % output_path)
except IOError as message:
print (message)
sys.exit(1)
except OSError as message:
print(message)
sys.exit(1)
except zipfile.BadZipfile as message:
print (message)
sys.exit(1)
finally:
zip_file.close()
Since I am new in python i cant explain the code in detail. Here are the references :
https://pypi.org/project/pyzipper/
https://www.calazan.com/how-to-zip-an-entire-directory-with-python/
To extract the Generated ZIP file in windows :
Right Click - > Unzip(Encripted)
If you directly click Extract All option, then it will give error

Try this:
Firstly check here please for pynzip. After that try it.
import pyminizip as pyzip
compression = 8
pyzip.compress("test.txt", "test.zip", "Pswrd", compression)

Here is how to copy all a directory with its subdirectories and its files, then compress it and encrypt a zip, with password and without needing an associated backup file, here we will see how to authorize a mac address to execute the decryption. So then it's up to you to change or improve the script.
But the essentials work very well.
After a lot of research, testing and thinking, I created this effective solution
my setup:
Python 3.8 64:bits on windows 7 64:bits
Usage terminology:
First step, we need to import the cryptography module
check for support or other is here https://cryptography.io/en/latest/installation/
command:
pip install cryptography
Then we will use the fernet object resulting from this module
https://cryptography.io/en/latest/fernet/
with password
https://cryptography.io/en/latest/fernet/#using-passwords-with-fernet
and shutil:
https://docs.python.org/3/library/shutil.html
file second.py:
import os
import re, uuid
import string
import shutil
import zlib
from cryptography.fernet import Fernet
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
import base64
import zipfile
class zipy:
def __init__(self, pathDir=None):
"""If pathDir optional is none, this script copy all directory in current execution."""
if pathDir != None:
if os.path.isdir(pathDir):
pathDir = pathDir.replace(os.sep, '/')
if pathDir.endswith('/'):
self.root = pathDir
else:
self.root = pathDir + '/'
else:
self.root = os.getcwd()+os.sep
self.root = self.root.replace(os.sep, '/')
else:
self.root = os.getcwd()+os.sep
self.root = self.root.replace(os.sep, '/')
os.chdir(self.root)
self.name = 'sauvegarde'
self.dirSauvegarde = self.root+self.name
self.dirSauvegarde = self.dirSauvegarde.replace(os.sep, '/')
lectureDossier = os.listdir(self.root)
print(lectureDossier)
self.path_system = {}
for element in lectureDossier:
if os.path.isdir(element):
if element != '__pycache__':
self.path_system[element] = self.root + element + os.sep.replace(os.sep, '/')
self.path_system[element] = self.path_system[element].replace(os.sep, '/')
else:
pass
elif os.path.isfile(element):
self.path_system[element] = self.root + element
self.path_system[element] = self.path_system[element].replace(os.sep, '/')
else:
pass
self.zipi = myZip(self.dirSauvegarde)
def save(self):
"""sauvegarde le fichier"""
self.createDir(self.dirSauvegarde)
chemin_src = ""
chemin_dist = ""
for element in self.path_system:
if element != self.dirSauvegarde:
chemin_src = self.root+element
chemin_dest = self.dirSauvegarde + os.sep + element
chemin_dest = chemin_dest.replace(os.sep, '/')
if os.path.isdir(chemin_src):
self.copyDir(chemin_src, chemin_dest)
else:
self.copyFile(chemin_src, chemin_dest)
self.zipi.zip(zip_exist=True)
self.delDir(self.dirSauvegarde)
def copyDir(self, src, dest):
try:
shutil.copytree(src, dest, dirs_exist_ok=True)
except:
pass
def copyFile(self, src, dest):
try:
shutil.copyfile(src, dest)
except:
pass
def createDir(self, dirPath):
if os.path.isdir(dirPath):
self.delDir(dirPath)
else:
pass
os.makedirs(dirPath, exist_ok=True)
def delDir(self, dir):
if os.path.isdir(dir):
if len(os.listdir(dir)) > 0:
try:
print('rmtree')
shutil.rmtree(dir, ignore_errors=True)
except:
pass
else:
try:
os.rmdir(dir)
except:
pass
def decrypt(self):
self.zipi.unzip()
class myZip:
def __init__(self, dir):
self.pathDir = dir
self.nom = os.path.basename(dir)
self.pathZip = self.pathDir + '.zip'
self.crypt = Encryptor()
def zip(self, zip_exist=False):
if zip_exist == False:
pass
else:
if os.path.isfile(self.pathZip):
try:
os.remove(self.pathZip)
except:
pass
shutil.make_archive(os.path.splitext(self.pathZip)[0], 'zip', self.pathDir)
key = self.crypt.key_create()
#TEST
self.crypt.file_encrypt(key, self.pathZip, self.pathZip)
self.crypt.key_write(self.pathZip, key)
def unzip(self):
#TEST
if self.crypt.checkPass(self.pathZip):
#print('ok adresse mac autoriser')
key = self.crypt.key_load(self.pathZip)
self.crypt.file_decrypt(key, self.pathZip, self.pathZip)
else:
print('pas ok adresse mac erroner')
class Encryptor:
def __init__(self):
self.salto = None
def key_create(self):
password = self.getMac()
password = bytes(password, encoding="utf-8")
self.salto = os.urandom(16)
print(self.salto)
kdf = PBKDF2HMAC(
algorithm=hashes.SHA256(),
length=32,
salt=self.salto,
iterations=100,
)
key = base64.urlsafe_b64encode(kdf.derive(password))
return key
def key_write(self, pathZip, key):
with zipfile.ZipFile(pathZip, 'a') as zip:
zip.comment = key + bytes(' byMe ', encoding="utf-8") + self.salto
def key_load(self, pathZip):
stri = []
with zipfile.ZipFile(pathZip, 'a') as zip:
stri = zip.comment.split(b' byMe ')
print(stri[0])
print(stri[1])
key = stri[0]
self.salto = stri[1]
return key
def checkPass(self, pathZip):
key = base64.urlsafe_b64decode(self.key_load(pathZip))
salt = self.salto
mdp = self.getMac()
mdp = bytes(mdp, encoding="utf-8")
kdf = PBKDF2HMAC(
algorithm=hashes.SHA256(),
length=32,
salt=salt,
iterations=100,
)
retour = False
try:
kdf.verify(mdp, key)
retour = True
except:
retour = False
return retour
def file_encrypt(self, key, original_file, encrypted_file):
f = Fernet(key)
with open(original_file, 'rb') as file:
original = file.read()
encrypted = f.encrypt(original)
with open (encrypted_file, 'wb') as file:
file.write(encrypted)
def file_decrypt(self, key, encrypted_file, decrypted_file):
f = Fernet(key)
with open(encrypted_file, 'rb') as file:
encrypted = file.read()
decrypted = f.decrypt(encrypted)
with open(decrypted_file, 'wb') as file:
file.write(decrypted)
def getMac(self):
return "".join(re.findall('..', '%012x' % uuid.getnode()))
Use like this:
file : main.py
from second import zipy
#If the argument is empty, the script will make a copy of the directory being executed, otherwise the script will work and output the zip in the place indicated in argument
dd = zipy("E:/path")
#or dd = zipy("E:/path/") or dd = zipy() if you give arg, give absolute path
#Save the zip and encrypt it. Change second.py to directly give it a password as an argument
dd.save()
#decrypt zip
dd.decrypt()

Here's a snippet with pyminizip: gets a list of files and zips the whole thing.
import pyminizip
import os
def get_paths_recursively(src_root_path):
files = []
if src_root_path is not None:
for root, directories, filenames in os.walk(src_root_path):
entries = []
for filename in filenames:
full_file_name = os.path.join(root, filename)
if os.path.isfile(full_file_name) and not filename.startswith('.'):
files.append(os.path.join(root, filename))
return files
def pyminizip_zipper(folder_path, output_path, password):
paths = get_paths_recursively(folder_path)
roots = []
for path in paths:
roots.append(os.path.dirname(path.replace(os.path.dirname(folder_path), './')))
pyminizip.compress_multiple(paths, roots, output_path, password, 5)

What is the right way to refer bazel data files in python?

Suppose I have the following BUILD file
py_library(
name = "foo",
src = ["foo.py"],
data = ["//bar:data.json"],
)
How should I refer to the data.json in foo.py file? I wanted to have something like below, what should I use for some_path?
with open(os.path.join(some_path, "bar/data.json"), 'r') as fp:
data = json.load(fp)
I couldn't find much general documentation about *.runfiles online -- any pointer will be appreciated!

Short answer: os.path.dirname(__file__)
Here is the full example:
$ ls
bar/ BUILD foo.py WORKSPACE
$ cat BUILD
py_binary(
name = "foo",
srcs = ["foo.py"],
data = ["//bar:data.json"],
)
$ cat foo.py
import json
import os
ws = os.path.dirname(__file__)
with open(os.path.join(ws, "bar/data.json"), 'r') as fp:
print(json.load(fp))
$ cat bar/BUILD
exports_files(["data.json"])
$ bazel run :foo
Edit: it doesn't work well when your package is in a subdirectory. You may need to go back using os.path.dirname.

Here is a function that should return the path to the runfiles root for any py_binary in all the cases that I'm aware of:
import os
import re
def find_runfiles():
"""Find the runfiles tree (useful when _not_ run from a zip file)"""
# Follow symlinks, looking for my module space
stub_filename = os.path.abspath(sys.argv[0])
while True:
# Found it?
module_space = stub_filename + '.runfiles'
if os.path.isdir(module_space):
break
runfiles_pattern = r"(.*\.runfiles)"
matchobj = re.match(runfiles_pattern, os.path.abspath(sys.argv[0]))
if matchobj:
module_space = matchobj.group(1)
break
raise RuntimeError('Cannot find .runfiles directory for %s' %
sys.argv[0])
return module_space
For the example in your question you could use it like so:
with open(os.path.join(find_runfiles(), "name_of_workspace/bar/data.json"), 'r') as fp:
data = json.load(fp)
Note that this function won't help if you build zipped executables of your python apps (using subpar, probably); for those you will need some more code. This next snippet includes get_resource_filename() and get_resource_directory(), which will work for both regular py_binary and .par binaries:
import atexit
import os
import re
import shutil
import sys
import tempfile
import zipfile
def get_resource_filename(path):
zip_path = get_zip_path(sys.modules.get("__main__").__file__)
if zip_path:
tmpdir = tempfile.mkdtemp()
atexit.register(lambda: shutil.rmtree(tmpdir, ignore_errors=True))
zf = BetterZipFile(zip_path)
zf.extract(member=path, path=tmpdir)
return os.path.join(tmpdir, path)
elif os.path.exists(path):
return path
else:
path_in_runfiles = os.path.join(find_runfiles(), path)
if os.path.exists(path_in_runfiles):
return path_in_runfiles
else:
raise ResourceNotFoundError
def get_resource_directory(path):
"""Find or extract an entire subtree and return its location."""
zip_path = get_zip_path(sys.modules.get("__main__").__file__)
if zip_path:
tmpdir = tempfile.mkdtemp()
atexit.register(lambda: shutil.rmtree(tmpdir, ignore_errors=True))
zf = BetterZipFile(zip_path)
members = []
for fn in zf.namelist():
if fn.startswith(path):
members += [fn]
zf.extractall(members=members, path=tmpdir)
return os.path.join(tmpdir, path)
elif os.path.exists(path):
return path
else:
path_in_runfiles = os.path.join(find_runfiles(), path)
if os.path.exists(path_in_runfiles):
return path_in_runfiles
else:
raise ResourceNotFoundError
def get_zip_path(path):
"""If path is inside a zip file, return the zip file's path."""
if path == os.path.sep:
return None
elif zipfile.is_zipfile(path):
return path
return get_zip_path(os.path.dirname(path))
class ResourceNotFoundError(RuntimeError):
pass
def find_runfiles():
"""Find the runfiles tree (useful when _not_ run from a zip file)"""
# Follow symlinks, looking for my module space
stub_filename = os.path.abspath(sys.argv[0])
while True:
# Found it?
module_space = stub_filename + '.runfiles'
if os.path.isdir(module_space):
break
runfiles_pattern = r"(.*\.runfiles)"
matchobj = re.match(runfiles_pattern, os.path.abspath(sys.argv[0]))
if matchobj:
module_space = matchobj.group(1)
break
raise RuntimeError('Cannot find .runfiles directory for %s' %
sys.argv[0])
return module_space
class BetterZipFile(zipfile.ZipFile):
"""Shim around ZipFile that preserves permissions on extract."""
def extract(self, member, path=None, pwd=None):
if not isinstance(member, zipfile.ZipInfo):
member = self.getinfo(member)
if path is None:
path = os.getcwd()
ret_val = self._extract_member(member, path, pwd)
attr = member.external_attr >> 16
os.chmod(ret_val, attr)
return ret_val
Using this second code snippet, your example would look like:
with open(get_resource_filename("name_of_workspace/bar/data.json"), 'r') as fp:
data = json.load(fp)