Develop Reference

Unexpected end of data when zipping zip files in Python

Good day.
I wrote a little Python program to help me easily create .cbc files for Calibre, which is just a renamed .zip file with a text file called comics.txt for TOC purposes. Each chapter is another zip file.
The issue is that the last zip file zipped always has the error "Unexpected end of data". The file itself is not corrupt, if I unzip it and rezip it it works perfectly. Playing around it seems that the problem is that Python doesn't close the last zip file after zipping it, since I can't delete the last zip while the program is still running since it's still open in Python. Needless to say, Calibre doesn't like the file and fails to convert it unless I manually rezip the affected chapters.
The code is as follows, checking the folders for not-image files, zipping the folders, zipping the zips while creating the text file, and "changing" extension.
import re, glob, os, zipfile, shutil, pathlib, gzip, itertools
Folders = glob.glob("*/")
items = len(Folders)
cn_list = []
cn_list_filtered = []
dirs_filtered = []
ch_id = ["c", "Ch. "]
subdir_im = []
total = 0
Dirs = next(os.walk('.'))[1]
for i in range(0, len(Dirs)):
for items in os.listdir("./" + Dirs[i]):
if items.__contains__('.png') or items.__contains__('.jpg'):
total+=1
else:
print(items + " not an accepted format.")
subdir_im.append(total)
total = 0
for fname in Folders:
if re.search(ch_id[0] + r'\d+' + r'[\S]' + r'\d+', fname):
cn = re.findall(ch_id[0] + "(\d+[\S]\d+)", fname)[0]
cn_list.append(cn)
elif re.search(ch_id[0] + r'\d+', fname):
cn = re.findall(ch_id[0] + "(\d+)", fname)[0]
cn_list.append(cn)
elif re.search(ch_id[1] + r'\d+' + '[\S]' + r'\d+', fname):
cn = re.findall(ch_id[1] + "(\d+[\S]\d+)", fname)[0]
cn_list.append(cn)
elif re.search(ch_id[1] + r'\d+', fname):
cn = re.findall(ch_id[1] + "(\d+)", fname)[0]
cn_list.append(cn)
else:
print('Warning: File found without proper filename format.')
cn_list_filtered = set(cn_list)
cn_list_filtered = sorted(cn_list_filtered)
cwd = os.getcwd()
Dirs = Folders
subdir_zi = []
total = 0
for i in range(0, len(cn_list_filtered)):
for folders in Dirs:
if folders.__contains__(ch_id[0] + cn_list_filtered[i] + " ")\
or folders.__contains__(ch_id[1] + cn_list_filtered[i] + " "):
print('Zipping folder ', folders)
namezip = "Chapter " + cn_list_filtered[i] + ".zip"
current_zip = zipfile.ZipFile(namezip, "a")
for items in os.listdir(folders):
if items.__contains__('.png') or items.__contains__('.jpg'):
current_zip.write(folders + "/" + items, items)
total+=1
subdir_zi.append(total)
total = 0
print('Folder contents in order:', subdir_im, ' Total:', sum(subdir_im))
print("Number of items per zip: ", subdir_zi, ' Total:', sum(subdir_zi))
if subdir_im == subdir_zi:
print("All items in folders have been successfully zipped")
else:
print("Warning: File count in folders and zips do not match. Please check the affected chapters")
zips = glob.glob("*.zip")
namezip2 = os.path.basename(os.getcwd()) + ".zip"
zipfinal = zipfile.ZipFile(namezip2, "a")
for i in range(0, len(zips), 1):
zipfinal.write(zips[i],zips[i])
Data = []
for i in range (0,len(cn_list_filtered),1):
Datai = ("Chapter " + cn_list_filtered[i] + ".zip" + ":Chapter " + cn_list_filtered[i] + "\r\n")
Data.append(Datai)
Dataok = ''.join(Data)
with zipfile.ZipFile(namezip2, 'a') as myzip:
myzip.writestr("comics.txt", Dataok)
zipfinal.close()
os.rename(namezip2, namezip2 + ".cbc")
os.system("pause")
I am by no means a programmer, that is just a Frankenstein monster code I eventually managed to put together by checking threads, but this last issue has me stumped.
Some solutions I tried are:
for i in range(0, len(zips), 1):
zipfinal.write(zips[i],zips[i])
zips[i].close()
Fails with:
zips[i].close()
AttributeError: 'str' object has no attribute 'close'
and:
for i in range(0, len(zips), 1):
zipfinal.write(zips[i],zips[i])
zips[len(zips)].close()
Fails with:
zips[len(zips)].close()
IndexError: list index out of range
Thanks for the help.

This solved my issue:
def generate_zip(file_list, file_name=None):
zip_buffer = io.BytesIO()
zf = zipfile.ZipFile(zip_buffer, mode="w", compression=zipfile.ZIP_DEFLATED)
for file in file_list:
print(f"Filename: {file[0]}\nData: {file[1]}")
zf.writestr(file[0], file[1])
**zf.close()**
with open(file_name, 'wb') as f:
f.write(zip_buffer.getvalue())
f.close()

How to check if folder has suboflders and then list the directories with listdir()?

I want to write a file searching code where I don't know if the directory I'm searching in has subdirectories and I want to check that so I don't get an error like this:
[Error 267]The directory name is invalid: 'C:/Path/To/Directory'.
I wrote a code like this where if it finds the file it breaks and stopps the program but if not it goes down a layer and so on.
filename = raw_input('> ')
path = 'C:/Path/Of/Directory/You/Want/To/Search/In'
fldr = os.listdir(path)
for f in fldr:
p = path + '/' + f
sfldr = os.listdir(p)
if os.path.exists(p + '/' + filename):
print 'Found file!!', p + '/' + filename
break
else:
for sf in sfldr:
pp = p + '/' + sf
ssfldr = os.listdir(pp)
if os.path.exists(pp + '/' + filename):
print 'Found file!!', pp + '/' + filename
break
else:
for ssf in ssfldr:
ppp = pp + '/' + ssf
sssfldr = os.listdir(ppp)
if os.path.exists(ppp + '/' + filename):
print 'Found file!!', ppp + '/' + filename
break
The easy to notice error is that when the directory doesn't have 3 layers of subfolders the program just breaks and gives an error message.So if I could check if the folder has subfolders before entering it,that would be neat.

Use os.scandir(). Provides better speed over os.walk()
Link to docs here!
Alternatively use, glob
>>> from glob import glob
>>> paths = glob('*/')
>>> paths
['bin/', 'content/', 'include/', 'lib/', 'output/']
>>>

Changing the foldername and file name under directory

I am trying to change the foldername, and the file names. They all have a $, and I want a #. Here is what I got
def cleanFiles(self):
#directory = '/Users/eeamesX/work/data/Sept_1_upload/priority_2/transcriptsAudoSplits/09012015_331_male3_r1_seg1/IT_007/hell$o '
directoryChosen = self.directoryChoice()
print directoryChosen + " you made it to files selected"
self.listWidget.addItem(directoryChosen)
#for file_names in os.listdir(directoryChosen):
#self.listWidget.addItem(file_names)
for n in os.listdir(directoryChosen):
print n + " made it here"
if os.path.isfile(directoryChosen):
print directoryChosen + "almost there"
newname = n.replace('$', '#')
print newname + " this is newname"
if newname != n:
os.rename(n,newname)
print '\n--------------------------------\n'
for n in os.listdir(directoryChosen):
print n
self.lblNamechange.show()
I have pinpointed my problem through the printing. It is in the line
if os.path.isfile(directoryChosen):
It is not reading the files in the directory to change the filenames. Any help? Also, Will this change the foldername?

If what you are passing is in fact a directory, then this:
if os.path.isfile(directoryChosen):
will never be True, because isfile checks whether what you are passing is in fact a file.
Running help(os.path.isfile) in your interpreter will show you:
isfile(path)
Test whether a path is a regular file
What you want to use is actually isdir:
if os.path.isdir(directoryChosen):

Converting multiple gz file within subdirectories into csv

I have many subdirectories in my main directory and would like to write a script to unzip and convert all the files within it. If possible, I would also like to combine all the CSV within a single directory into a single CSV. But more importantly, I need help with my nested loop.
import gzip
import csv
import os
subdirlist = os.listdir('/home/user/Desktop/testloop')
subtotal = len(subdirlist)
subcounter = 0
for dirlist in subdirlist:
print "Working On " + dirlist
total = len(dirlist)
counter = 0
for dir in dirlist:
print "Working On " + dir
f = gzip.open('/' + str(subdirlist) + '/' + dir, 'rb')
file_content = f.read()
f.close()
print "25% Complete"
filename = '/' + str(subdirlist) + '/temp.txt'
target = open(filename, 'w')
target.write(file_content)
target.close()
print "50% Complete!"
csv_file = '/' + str(subdirlist) + '/' + str(dir) + '.csv'
in_txt = csv.reader(open(filename, "rb"), delimiter = '\t')
out_csv = csv.writer(open(csv_file, 'wb'))
out_csv.writerows(in_txt)
os.remove(filename)
os.remove('/' + str(subdirlist) + '/' + dir)
counter+=1
print str(counter) + "/" + str(total) + " " + str(dir) + " Complete!"
print "SubDirectory Converted!"
print str(subcounter) + "/" + str(subtotal) + " " + str(subdirlist) + " Complete!"
subcounter+=1
print "All Files Converted!"
Thanks in advance

To get lists of files and subdirectories, you can use os.walk. Below is an implementation I wrote to get all files (optionally, of certain type(s)) in arbitrarily nested subdirectories:
from os import walk, sep
from functools import reduce # in Python 3.x only
def get_filelist(root, extensions=None):
"""Return a list of files (path and name) within a supplied root directory.
To filter by extension(s), provide a list of strings, e.g.
get_filelist(root, ["zip", "csv"])
"""
return reduce(lambda x, y: x+y,
[[sep.join([item[0], name]) for name in item[2]
if (extensions is None or
name.split(".")[-1] in extensions)]
for item in walk(root)])

python function not repeating

I have a python tool to 'touch' (utime) a file, then move to another folder. However, if the file already exists in the destination folder, it is silently overwritten. I would like to check for a file of the same name in the destination folder and, if it exists, rename the one I am moving to its name plus '-n' at the end, where n is a number starting at '1' and, if the file with '-1' at the end already exists, '-2' etc.
For example, say in the source folder is a file 'foo.txt', but there is one 'foo.txt' in the destination folder as well. This function should return '(absolute path)/foo-1.txt'.
So, I have made a function to check for these circumstances and return the modified string, so I can use rename later and not overwrite. However, currently, if the file exists, it returns nothing. Following is the function code, assuming these vars:
fileName - input filepath, absolute path. e.g. /Users/foo/sourceFolder/bar.txt
index - iterator variable, set to '1' at the start of each file being opened (externally to function)
def checkExists(fileName):
global index
print "checkExists(" + fileName + ")"
if exists(fileName):
splitPath = split(newFile)
splitName = splitext(splitPath[1])
newSplitName = splitName[0] + "-" + str(index)
index += 1
newName = splitPath[0] + "/" + newSplitName + splitName[1]
print "newName = " + newName
else:
print "(else) fileName = " + fileName
print "(else) fileName = " + str(type(fileName))
print ""
return fileName
checkExists(newName)
Now it seems that the inner call for checkExists() at the end is not running.
I hope I have been clear in my explanation.
IAmThePiGuy
P.S. I do not want to hear about potential race problems with utime, I know that the files in the source directory will not be accessed by anything else.

Your problem is that, if the file exists, you don't return anything. I think you're intending to use recursion to check the new filename, but you don't return the result of that call. Here's a stab:
def checkExists(fileName, index=0):
print "checkExists(" + fileName + ")"
if exists(fileName):
splitPath = split(newFile)
splitName = splitext(splitPath[1])
newSplitName = splitName[0] + "-" + str(index)
index += 1
newName = splitPath[0] + "/" + newSplitName + splitName[1]
print "newName = " + newName
return checkExists(newName, index) # recurse
else:
print "(else) fileName = " + fileName
print "(else) fileName = " + str(type(fileName))
print ""
return fileName
I also took the liberty of moving the recursive call closer to the generation of newName and removing the global variable and replacing that with recursion as well.

Here's an iterative approach to a similar problem:
def copyfile(path, dstdir, verbose=True, dryrun=False):
"""Copy `path` file to `dstdir` directory incrementing name if necessary."""
filename = os.path.basename(path)
basename, ext = os.path.splitext(filename)
for i in itertools.count(2):
destpath = os.path.join(dstdir, filename)
if not os.path.exists(destpath):
if verbose:
print(path, '->', destpath)
if not dryrun:
shutil.copyfile(path, destpath)
return
# increment filename
filename = "%s_%02d%s" % (basename, i, ext)