Develop Reference

Python - How to create a csv, if csv already exists [duplicate]

Does Python have any built-in functionality to add a number to a filename if it already exists?
My idea is that it would work the way certain OS's work - if a file is output to a directory where a file of that name already exists, it would append a number or increment it.
I.e: if "file.pdf" exists it will create "file2.pdf", and next time "file3.pdf".

I ended up writing my own simple function for this. Primitive, but gets the job done:
def uniquify(path):
filename, extension = os.path.splitext(path)
counter = 1
while os.path.exists(path):
path = filename + " (" + str(counter) + ")" + extension
counter += 1
return path

In a way, Python has this functionality built into the tempfile module. Unfortunately, you have to tap into a private global variable, tempfile._name_sequence. This means that officially, tempfile makes no guarantee that in future versions _name_sequence even exists -- it is an implementation detail.
But if you are okay with using it anyway, this shows how you can create uniquely named files of the form file#.pdf in a specified directory such as /tmp:
import tempfile
import itertools as IT
import os
def uniquify(path, sep = ''):
def name_sequence():
count = IT.count()
yield ''
while True:
yield '{s}{n:d}'.format(s = sep, n = next(count))
orig = tempfile._name_sequence
with tempfile._once_lock:
tempfile._name_sequence = name_sequence()
path = os.path.normpath(path)
dirname, basename = os.path.split(path)
filename, ext = os.path.splitext(basename)
fd, filename = tempfile.mkstemp(dir = dirname, prefix = filename, suffix = ext)
tempfile._name_sequence = orig
return filename
print(uniquify('/tmp/file.pdf'))

I was trying to implement the same thing in my project but #unutbu's answer seemed too 'heavy' for my needs so I came up with following code finally:
import os
index = ''
while True:
try:
os.makedirs('../hi'+index)
break
except WindowsError:
if index:
index = '('+str(int(index[1:-1])+1)+')' # Append 1 to number in brackets
else:
index = '(1)'
pass # Go and try create file again
Just in case someone stumbled upon this and requires something simpler.

If all files being numbered isn't a problem, and you know beforehand the name of the file to be written, you could simply do:
import os
counter = 0
filename = "file{}.pdf"
while os.path.isfile(filename.format(counter)):
counter += 1
filename = filename.format(counter)

recently I encountered the same thing and here is my approach:
import os
file_name = "file_name.txt"
if os.path.isfile(file_name):
expand = 1
while True:
expand += 1
new_file_name = file_name.split(".txt")[0] + str(expand) + ".txt"
if os.path.isfile(new_file_name):
continue
else:
file_name = new_file_name
break

Let's say you already have those files:
This function generates the next available non-already-existing filename, by adding a _1, _2, _3, ... suffix before the extension if necessary:
import os
def nextnonexistent(f):
fnew = f
root, ext = os.path.splitext(f)
i = 0
while os.path.exists(fnew):
i += 1
fnew = '%s_%i%s' % (root, i, ext)
return fnew
print(nextnonexistent('foo.txt')) # foo_3.txt
print(nextnonexistent('bar.txt')) # bar_1.txt
print(nextnonexistent('baz.txt')) # baz.txt

Since the tempfile hack A) is a hack and B) still requires a decent amount of code anyway, I went with a manual implementation. You basically need:
A way to Safely create a file if and only if it does not exist (this is what the tempfile hack affords us).
A generator for filenames.
A wrapping function to hide the mess.
I defined a safe_open that can be used just like open:
def iter_incrementing_file_names(path):
"""
Iterate incrementing file names. Start with path and add " (n)" before the
extension, where n starts at 1 and increases.
:param path: Some path
:return: An iterator.
"""
yield path
prefix, ext = os.path.splitext(path)
for i in itertools.count(start=1, step=1):
yield prefix + ' ({0})'.format(i) + ext
def safe_open(path, mode):
"""
Open path, but if it already exists, add " (n)" before the extension,
where n is the first number found such that the file does not already
exist.
Returns an open file handle. Make sure to close!
:param path: Some file name.
:return: Open file handle... be sure to close!
"""
flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY
if 'b' in mode and platform.system() == 'Windows':
flags |= os.O_BINARY
for filename in iter_incrementing_file_names(path):
try:
file_handle = os.open(filename, flags)
except OSError as e:
if e.errno == errno.EEXIST:
pass
else:
raise
else:
return os.fdopen(file_handle, mode)
# Example
with safe_open("some_file.txt", "w") as fh:
print("Hello", file=fh)

I haven't tested this yet but it should work, iterating over possible filenames until the file in question does not exist at which point it breaks.
def increment_filename(fn):
fn, extension = os.path.splitext(path)
n = 1
yield fn + extension
for n in itertools.count(start=1, step=1)
yield '%s%d.%s' % (fn, n, extension)
for filename in increment_filename(original_filename):
if not os.isfile(filename):
break

This works for me.
The initial file name is 0.yml, if it exists, it will add one until meet the requirement
import os
import itertools
def increment_filename(file_name):
fid, extension = os.path.splitext(file_name)
yield fid + extension
for n in itertools.count(start=1, step=1):
new_id = int(fid) + n
yield "%s%s" % (new_id, extension)
def get_file_path():
target_file_path = None
for file_name in increment_filename("0.yml"):
file_path = os.path.join('/tmp', file_name)
if not os.path.isfile(file_path):
target_file_path = file_path
break
return target_file_path

import os
class Renamer():
def __init__(self, name):
self.extension = name.split('.')[-1]
self.name = name[:-len(self.extension)-1]
self.filename = self.name
def rename(self):
i = 1
if os.path.exists(self.filename+'.'+self.extension):
while os.path.exists(self.filename+'.'+self.extension):
self.filename = '{} ({})'.format(self.name,i)
i += 1
return self.filename+'.'+self.extension

I found that the os.path.exists() conditional function did what I needed. I'm using a dictionary-to-csv saving as an example, but the same logic could work for any file type:
import os
def smart_save(filename, dict):
od = filename + '_' # added underscore before number for clarity
for i in np.arange(0,500,1): # I set an arbitrary upper limit of 500
d = od + str(i)
if os.path.exists(d + '.csv'):
pass
else:
with open(d + '.csv', 'w') as f: #or any saving operation you need
for key in dict.keys():
f.write("%s,%s\n"%(key, dictionary[key]))
break
Note: this appends a number (starting at 0) to the file name by default, but it's easy to shift that around.

This function validates if the file name exists using regex expresion and recursion
def validate_outfile_name(input_path):
filename, extension = os.path.splitext(input_path)
if os.path.exists(input_path):
output_path = ""
pattern = '\([0-9]\)'
match = re.search(pattern, filename)
if match:
version = filename[match.start() + 1]
try: new_version = int(version) + 1
except: new_version = 1
output_path = f"{filename[:match.start()]}({new_version}){extension}"
output_path = validate_outfile_name(output_path)
else:
version = 1
output_path = f"{filename}({version}){extension}"
return output_path
else:
return input_path

I've implemented a similar solution with pathlib:
Create file-names that match the pattern path/<file-name>-\d\d.ext. Perhaps this solution can help...
import pathlib
from toolz import itertoolz as itz
def file_exists_add_number(path_file_name, digits=2):
pfn = pathlib.Path(path_file_name)
parent = pfn.parent # parent-dir of file
stem = pfn.stem # file-name w/o extension
suffix = pfn.suffix # NOTE: extension starts with '.' (dot)!
try:
# search for files ending with '-\d\d.ext'
last_file = itz.last(parent.glob(f"{stem}-{digits * '?'}{suffix}"))
except:
curr_no = 1
else:
curr_no = int(last_file.stem[-digits:]) + 1
# int to string and add leading zeros
curr_no = str(last_no).zfill(digits)
path_file_name = parent / f"{stem}-{curr_no}{suffix}"
return str(path_file_name)
Pls note: That solution starts at 01 and will only find file-pattern containing -\d\d!

def create_file():
counter = 0
filename = "file"
while os.path.isfile(f"dir/{filename}{counter}.txt"):
counter += 1
print(f"{filename}{counter}.txt")

A little bit later but there is still something like this should work properly, mb it will be useful for someone.
You can use built-in iterator to do this ( image downloader as example for you ):
def image_downloader():
image_url = 'some_image_url'
for count in range(10):
image_data = requests.get(image_url).content
with open(f'image_{count}.jpg', 'wb') as handler:
handler.write(image_data)
Files will increment properly. Result is:
image.jpg
image_0.jpg
image_1.jpg
image_2.jpg
image_3.jpg
image_4.jpg
image_5.jpg
image_6.jpg
image_7.jpg
image_8.jpg
image_9.jpg

Easy way for create new file if this name in your folder
if 'sample.xlsx' in os.listdir('testdir/'):
i = 2
while os.path.exists(f'testdir/sample ({i}).xlsx'):
i += 1
wb.save(filename=f"testdir/sample ({i}).xlsx")
else:
wb.save(filename=f"testdir/sample.xlsx")

Check if a string contains any file extension whatsoever

I'm sure this is a simple thing to do but I don't know how. What I want to achieve is something like this:
templateFilename = str( templateFilename )
# If no file extension is found, assume it is a .npy file
if templateFilename.endswith( '.*' ):
templateFilename += ".npy"
However, this syntax doesn't seem to work. I want the * to represent any file extension so that, if the parsed file does contain a file extension, that one will be used but, if not, a standard extension will be added.
I have read about the glob module and people seem to be using that for finding things such as *.txt, etc. but I'm not sure how it works.

I would suggest os.path.splitext. The following uses .npy as the extension if none exists:
root, ext = os.path.splitext(path)
if not ext:
ext = '.npy'
path = root + ext

(Speaking from experience and hair-loss)
Doing a split on . and then selecting the second element [1] will only work if you can absolutely guarantee that there are no . in the filename; otherwise you'll need something like this:
file_extension = [".csv", ".xml", ".html"]
if '.' in templateFilename: #checks if you can actually split, if you can't perform a split; you would raise an index error.
if templateFilename.split(".")[-1] in file_extension: #[-1] = the last element in the list.
has_extension = true
has_verified_extension = true
else:
has_extension = true
has_verified_extension = false
else: #no '.'. in the filename, so no extension.
has_extension = false

Usage:
file_extension = [".pyo", ".npy", ".py"]
templateFilename = str( templateFilename )
# If no file extension is found, assume it is a .npy file
if not templateFilename.split(".")[1] in file_extension:
templateFilename += ".npy"

If you want in one line then here it is :
templatefilename = "abcd"
non_ext_file_list = [filename + ".npy" for filename in templateFilename.split(".") if not "." in templateFilename]
#output
[abcd.npy]

Move file to a folder or make a renamed copy if it exists in the destination folder

I have a piece of code i wrote for school:
import os
source = "/home/pi/lab"
dest = os.environ["HOME"]
for file in os.listdir(source):
if file.endswith(".c")
shutil.move(file,dest+"/c")
elif file.endswith(".cpp")
shutil.move(file,dest+"/cpp")
elif file.endswith(".sh")
shutil.move(file,dest+"/sh")
what this code is doing is looking for files in a source directory and then if a certain extension is found the file is moved to that directory. This part works. If the file already exists in the destination folder of the same name add 1 at end of the file name, and before the extension and if they are multiples copies do "1++".
Like this: test1.c,test2.c, test3.c
I tried using os.isfile(filename) but this only looks at the source directory. and I get a true or false.

To test if the file exists in the destination folder you should os.path.join the dest folder with the file name
import os
import shutil
source = "/home/pi/lab"
dest = os.environ["HOME"]
# Avoid using the reserved word 'file' for a variable - renamed it to 'filename' instead
for filename in os.listdir(source):
# os.path.splitext does exactly what its name suggests - split the name and extension of the file including the '.'
name, extension = os.path.splitext(filename)
if extension == ".c":
dest_filename = os.path.join(dest, filename)
if not os.path.isfile(dest_filename):
# We copy the file as is
shutil.copy(os.path.join(source, filename) , dest)
else:
# We rename the file with a number in the name incrementing the number until we find one that is not used.
# This should be moved to a separate function to avoid code duplication when handling the different file extensions
i = 0
dest_filename = os.path.join(dest, "%s%d%s" % (name, i, extension))
while os.path.isfile(dest_filename):
i += 1
dest_filename = os.path.join(dest, "%s%d%s" % (name, i, extension))
shutil.copy(os.path.join(source, filename), dest_filename)
elif extension == ".cpp"
...
# Handle other extensions
If you want to have put the renaming logic in a separate function using glob and re this is one way:
import glob
import re
...
def rename_file(source_filename, source_ext):
filename_pattern = os.path.join(dest, "%s[0-9]*%s"
% (source_filename, source_ext))
# Contains file such as 'a1.c', 'a2.c', etc...
existing_files = glob.glob(filename_pattern)
regex = re.compile("%s([0-9]*)%s" % (source_filename, source_ext))
# Retrieve the max of the index used for this file using regex
max_index = max([int(match.group(1))
for match in map(regex.search, existing_files)
if match])
source_full_path = os.path.join(source, "%s%s"
% (source_filename, source_ext))
# Rebuild the destination filename with the max index + 1
dest_full_path = os.path.join(dest, "%s%d%s"
% (source_filename,
(max_index + 1),
source_ext))
shutil.copy(source_full_path, dest_full_path)
...
# If the file already exists i.e. replace the while loop in the else statement
rename_file(name, extension)

I din't test the code. But something like this should do the job:-
i = 0
filename = "a.txt"
while True:
if os.isfile(filename):
i+= 1
break
if i:
fname, ext = filename.split('.')
filename = fname + str(i) + '.' + ext

Recursively renaming directory/file structures on a local file system

I'm trying to define arg1 outside of rename() and it does not work since dirs is not defined. If I use rename("dirs", False), the function does not work.
Any idea?
# Defining the function that renames the target
def rename(arg1, arg2):
for root, dirs, files in os.walk( # Listing
path, topdown=arg2):
for i, name in enumerate(arg1):
output = name.replace(pattern, "") # Taking out pattern
if output != name:
os.rename( # Renaming
os.path.join(root, name),
os.path.join(root, output))
else:
pass
# Run
rename(dirs, False)
Here's the whole program:
#!/usr/bin/python
# -*- coding: utf-8 -*-
# This program batch renames files or folders by taking out a certain pattern
import os
import subprocess
import re
# Defining the function that renames the target
def rename(arg1, arg2):
for root, dirs, files in os.walk( # Listing
path, topdown=arg2):
for i, name in enumerate(arg1):
output = name.replace(pattern, "") # Taking out pattern
if output != name:
os.rename( # Renaming
os.path.join(root, name),
os.path.join(root, output))
else:
pass
# User chooses between file and folder
print "What do you want to rename?"
print "1 - Folders\n2 - Files\n"
valid = False
while not valid:
try:
choice = int(raw_input("Enter number here: "))
if choice > 2:
print "Please enter a valid number\n"
valid = False
else:
valid = True
except ValueError:
print "Please enter a valid number\n"
valid = False
choice = 3 # To have a correct value of choice
# Asking for path & pattern
if choice == 1:
kind = "folders"
elif choice == 2:
kind = "files"
else:
pass
path = raw_input("What is the path to the %s?\n " % (kind))
pattern = raw_input("What is the pattern to remove?\n ")
# CHOICE = 1
# Renaming folders
if choice == 1:
rename(dirs, False)
# CHOICE = 2
# Renaming files
if choice == 2:
rename(files,True)
# Success message
kind = kind.replace("f", "F")
print "%s renamed" % (kind)

Recorrect my code in a better way.
#!/usr/bin/env python
import os
import sys
# the command like this: python rename dirs /your/path/name/ tst
if __name__ == '__main__':
mode = sys.argv[1] # dirs or files
pathname = sys.argv[2]
pattern = sys.argv[3]
ndict = {'dirs': '', 'files': ''}
topdown = {'dirs': False, 'files': True}
for root, ndict['dirs'], ndict['files'] in os.walk(
pathname, topdown[mode]):
for name in enumerate(ndict[mode]):
newname = name.replace(pattern, '')
if newname != name:
os.rename(
os.path.join(root, name),
os.path.join(root, newname))

This is better achieved as a command-line tool using the py library:
import sys
from py.path import local # import local path object/class
def rename_files(root, pattern):
"""
Iterate over all paths starting at root using ``~py.path.local.visit()``
check if it is a file using ``~py.path.local.check(file=True)`` and
rename it with a new basename with ``pattern`` stripped out.
"""
for path in root.visit(rec=True):
if path.check(file=True):
path.rename(path.new(basename=path.basename.replace(pattern, "")))
def rename_dirs(root, pattern):
"""
Iterate over all paths starting at root using ``~py.path.local.visit()``
check if it is a directory using ``~py.path.local.check(dir=True)`` and
rename it with a new basename with ``pattern`` stripped out.
"""
for path in root.visit(rec=True):
if path.check(dir=True):
path.rename(path.new(basename=path.basename.replace(pattern, "")))
def main():
"""Define our main top-level entry point"""
root = local(sys.argv[1]) # 1 to skip the program name
pattern = sys.argv[2]
if local(sys.argv[0]).purebasename == "renamefiles":
rename_files(root, pattern)
else:
rename_dirs(root, pattern)
if __name__ == "__main__":
"""
Python sets ``__name__`` (a global variable) to ``__main__`` when being called
as a script/application. e.g: Python renamefiles or ./renamefiles
"""
main() # Call our main function
Usage:
renamefiles /path/to/dir pattern
or:
renamedirs /path/to/dir pattern
Save this as renamefiles or renamedirs.
A common approach in UNIX is to name the script/tool renamefiles and symlink renamefiles to renamedirs.
Improvement Notes:
Use optparse or argparse to provide Command Line Options = and a --help
Make rename_files() and rename_dirs() generic and move it into a single function.
Write documentation (docstrings)
Write unit tests.

Comparing two directories with subdirectories to find any changes?

For starters I've only been playing with python for about a 2 weeks now and im relatively new to its proccessess, I'm trying to create a script that compares two directories with subdirectories and prints out ANY changes. I've read articles on hear about using os.walk to walk the directories and I've managed to write the script that prints all the files in a directory and its subdirectories in a understandable manner. I've also read on here and learned how to compare two directories but it only compares 1 file deep.
import os
x = 'D:\\xfiles'
y = 'D:\\yfiles'
q= [ filename for filename in x if filename not in y ]
print q
Obviously that does not do what I want it to. This however is listing all files and all directories.
import os
x = 'D:\\xfiles'
x1 = os.walk(x)
for dirName, subdirList, fileList in x1:
print ('Directory: %s' % dirName)
for fname in fileList:
print ('\%s' % fname)
How do I combine them and get it to work?

Write a function to aggregate your listing.
import os
def listfiles(path):
files = []
for dirName, subdirList, fileList in os.walk(path):
dir = dirName.replace(path, '')
for fname in fileList:
files.append(os.path.join(dir, fname))
return files
x = listfiles('D:\\xfiles')
y = listfiles('D:\\yfiles')
You could use a list comprehension to extract the files that are not in both directories.
q = [filename for filename in x if filename not in y]
But using sets is much more efficient and flexible.
files_only_in_x = set(x) - set(y)
files_only_in_y = set(y) - set(x)
files_only_in_either = set(x) ^ set(y)
files_in_both = set(x) & set(y)
all_files = set(x) | set(y)

I guess that best way to go will be external programs, as #Robᵩ suggests in the comment.
Using Python I would recommend doing following:
import os
def fileIsSame(right, left, path):
return os.path.exists (os.path.join(left, path.replace(right, '')));
def compare(right, left):
difference = list();
for root, dirs, files in os.walk(right):
for name in files:
path = os.path.join(root, name);
# check if file is same
if fileIsSame(right, left, path):
if os.path.isdir(path):
# recursively check subdirs
difference.extend(compare(path, left));
else:
# count file as difference
difference.append(path);
return difference;
This approach lacks normal fileIsSame function that would make sure files are same by content or by date modified and be sure to handle paths correctly (as I'm not sure this variant will). This algorithm requres you to specify full paths.
Usage example:
print (compare(r'c:\test', r'd:\copy_of_test'));
If second folder is copy of first, all the differences in paths (different disk letter and foldername) is ignored. Output will be [].

import os
def ls(path):
all = []
walked = os.walk(path)
for base, sub_f, files in walked:
for sub in sub_f:
entry = os.path.join(base,sub)
entry = entry[len(path):].strip("\\")
all.append(entry)
for file in files:
entry = os.path.join(base,file)
entry = entry[len(path):].strip("\\")
all.append(entry)
all.sort()
return all
def folder_diff(folder1_path, folder2_path):
folder1_list = ls(folder1_path);
folder2_list = ls(folder2_path);
diff = [item for item in folder1_list if item not in folder2_list]
diff.extend([item for item in folder2_list if item not in folder1_list])
return diff

I have done a code that check two directory recursively, and if there is different, it would point out the line that different.
import os
FOLDER_A = os.path.join(os.path.dirname(__file__), 'folder_a')
FOLDER_B = os.path.join(os.path.dirname(__file__), 'folder_b')
def load_directory(directory):
files = set()
directories = set()
for file_or_directory in os.listdir(directory):
file_or_directory_path = f'{directory}/{file_or_directory}'
if os.path.isfile(file_or_directory_path):
files.add(file_or_directory)
else:
directories.add(file_or_directory)
return files, directories
def compare_files(a, b):
assert os.path.isfile(a)
assert os.path.isfile(b)
with open(a, 'r') as file:
file_a = file.read()
with open(b, 'r') as file:
file_b = file.read()
if file_a != file_b:
file_a_lines = file_a.split('\n')
file_b_lines = file_b.split('\n')
if len(file_a_lines) != len(file_b_lines):
print(f'Two file {a} and {b} have different length, of {len(file_a_lines)} and {len(file_b_lines)}')
return False
compare_lines = zip(file_a_lines, file_b_lines)
index = 0
for i in compare_lines:
index += 1
if i[0] != i[1]:
print(f'Different found in file {a} and {b}, at line number {index}')
return False
print('Some thing wrong')
return False
return True
def compare_directories(a, b):
assert not os.path.isfile(a)
assert not os.path.isfile(b)
a_files, a_directories = load_directory(a)
b_files, b_directories = load_directory(b)
if (a_files != b_files):
print(f'Different Found In {a} and {b} directories files')
print(f'A: {a_files}\nB: {b_files}')
return False
if (a_directories != b_directories):
print(f'Different Found In {a} and {b} directories subdirectories')
print(f'A: {a_directories}\nB: {b_directories}')
return False
for files in a_files:
if not compare_files(f'{a}/{files}', f'{b}/{files}'):
return False
for directories in a_directories:
if not compare_directories(f'{a}/{directories}', f'{b}/{directories}'):
return False
return True
def main():
print(compare_directories(FOLDER_A, FOLDER_B))
if __name__ == '__main__':
main()