Develop Reference

Copy files from multiple specific subfolders

Under the file path D:/src, I have images folders and its subfolders which have a regular structure as follows:
Folder A
- Subfolder a
- Subfolder b
- Subfolder c
Folder B
- Subfolder a
- Subfolder b
- Subfolder c
- Subfolder d
Folder C
- Subfolder a
- Subfolder b
- Subfolder c
...
I want to copy all .jpg files in Subfolder b from Folder A, B, C and so on to a new folder Subfolder b in D:/dst. How can I do it in Python? Thanks.
Subfolder b
-xxx.jpg
-xyx.jpg
-yxz.jpg
...
Here is what I have found from the following link may helps:
Copy certain files from one folder to another using python
import os;
import shutil;
import glob;
source="C:/Users/X/Pictures/test/Z.jpg"
dest="C:/Users/Public/Image"
if os.path.exists(dest):
print("this folder exit in this dir")
else:
dir = os.mkdir(dest)
for file in glob._iglob(os.path.join(source),""):
shutil.copy(file,dest)
print("done")

Try this
import os
from os.path import join, isfile
BASE_PATH = 'd:/test'
SUBFOLDER = 'Subfolder b'
for folder, subfolders, *_ in os.walk(BASE_PATH):
if SUBFOLDER in subfolders:
full_path = join(BASE_PATH, folder, SUBFOLDER)
files = [f for f in os.listdir(full_path) if isfile(join(full_path, f)) and f.lower().endswith(('.jpg', '.jpeg'))]
for f in files:
file_path = join(full_path, f)
print (f'Copy {f} somewehere')

Assuming you have 2 levels of nesting
root_dir = './data'
dest_dir = './new_location'
os.listdir(root_dir)
for folder in os.listdir(root_dir):
folder_path = os.path.join(root_dir, folder)
if os.path.isdir(folder_path):
for subfolder in os.listdir(folder_path):
if subfolder == 'Subfolder B':
subfolder_path = os.path.join(root_dir, folder, subfolder)
print(subfolder_path)
for filename in os.listdir(subfolder_path):
file_path = os.path.join(root_dir, folder, subfolder, filename)
dest_path = os.path.join(dest_dir, filename)
shutil.copy(file_path, dest_path)
print("Copied ", file_path, "to", dest_path)
You just need 2 for loops, and in the the inner for loop you just check whether the folder name matches Subfolder B. If it does then copy all the files inside that directory to your destination folder.

Here's a short script that should do the work...
import os
# list all the directories in current directory
dirs = [x[0] for x in os.walk("D:/src")]
for d in dirs:
## list all files in A/b/*, B/b/*, C/b/*...
files_to_copy = os.listdir(os.path.join(d, "b"))
for f in files_to_copy:
if f.endswith(".jpg"): ## copy the relevant files to dest
shutil.copy(os.path.join(d, "b", f), os.path.join(dest, f))

import shutil
import os
def move_files(source_director, destination_director):
# select the files in the source directory
fisiere = [fisier for fisier in os.listdir(source_director) if os.path.isfile(os.path.join(source_director, fisier))]
print(fisiere)
# each file in the source directory is moved to the destination directory
for fisier in fisiere:
shutil.move(source_director + '\\' + fisier, destination_director)
def copy_files(source_director, destination_director):
# select the files in the source directory
fisiere = [fisier for fisier in os.listdir(source_director) if os.path.isfile(os.path.join(source_director, fisier))]
print(fisiere)
# each file in the source directory is copied to the destination directory
for fisier in fisiere:
shutil.copy(source_director + '\\' + fisier, destination_director)
if __name__ == '__main__':
# WARNING: paths to directories must be written with '\\' (double backslash)
# here you set the DIN directory you want to move / copy the files
source_director = 'c:\\Folder'
destination_director = 'c:\\Folder\\Subfolder1\\Subfolder2'
#CASE 1
# move_files(source_director, destination_director)
#CASE 2
copy_files(source_director, destination_director)
THE SOURCE OF THIS CODE HERE:
https://neculaifantanaru.com/en/python-copy-or-moves-files-from-a-folder-to-another-folder.html

Python: How to get the full path of a file in order to move it?

I had files that were in zips. I unzipped them with Zip-7 so they are in folders with the zip file names.
Each of these folders has either a .otf or .ttf (some have both) that I want out of them and moved to another folder.
I have tried a few methods of getting the full path of the files but every one of them leaves out the folder that the file is actually in.
Here is my latest try:
import os
import shutil
from pathlib import Path
result = []
for root, dirs, files in os.walk("."):
for d in dirs:
continue
for f in files:
if f.endswith(".otf"):
print(f)
p = Path(f).absolute()
parent_dir = p.parents[1]
p.rename(parent_dir / p.name)
elif f.endswith(".ttf"):
print(f)
p = Path(f).absolute()
parent_dir = p.parents[1]
p.rename(parent_dir / p.name)
else:
continue
Other attempts:
# parent_dir = Path(f).parents[1]
# shutil.move(f, parent_dir)
#print("OTF: " + f)
# fn = f
# f = f[:-4]
# f += '\\'
# f += fn
# result.append(os.path.realpath(f))
#os.path.relpath(os.path.join(root, f), "."))
I know this is something simple but I just can't figure it out. Thanks!

You should join the file name with the path name root:
for root, dirs, files in os.walk("."):
for d in dirs:
continue
for f in files:
if f.endswith(".otf"):
p = Path(os.path.join(root, f)).absolute()
parent_dir = p.parents[1]
p.rename(parent_dir / p.name)
elif f.endswith(".ttf"):
p = Path(os.path.join(root, f)).absolute()
parent_dir = p.parents[1]
p.rename(parent_dir / p.name)
else:
continue

for root, dirs, files in os.walk(".")
for d in dirs:
continue
for f in files:
print(os.path.abspath(f))
You can use os.path.abspath() to get a path of a full file
You would also need to still filter for the certain file types.

Patch Movement: File Backup and Replace

I'm getting an error while trying to copy files from a single source directory which contains bug fixes: /home/saurabh/testbed/patch_dir/ to multiple destination directories: app_dir_1 and app_dir_2.
Both these directories are exact replicas of each other.
The script does the following:-
Read lines from a text file into a list. Each line contains the name of one component. In this case: ['file1.class', file2.html]
Search value at each index recursively, starting from a particular directory:
/home/saurabh/testbed/dest_dir/
Take a backup of these files wherever they are found by appending ddMonyyyy to their extension.
Copy files from directory which contains patched components: /home/saurabh/testbed/patch_dir/
to the directory where backup was taken earlier
Directory Overview:-
/home/saurabh/testbed/dest_dir/
|--app_dir_1
|--file1.class
|--file2.html
|--file3.jsp
|--file4.xml
|--sub_dir
|--app_dir_2
|--file1.class
|--file2.html
|--file3.jsp
|--file4.xml
|--sub_dir
|--other_directories
/home/saurabh/testbed/patch_dir/
|--file1.class
|--file2.html
Below is my code:
#!/usr/bin/env python
import os
import fnmatch
import datetime
import shutil
with open('filenames.txt') as f:
content = f.readlines()
content = [x.strip() for x in content]
print('File contents:')
print(content)
suffix = datetime.datetime.now().strftime("_%d%b%Y")
approot = '/home/saurabh/testbed/dest_dir/'
source_dir = '/home/saurabh/testbed/patch_dir/'
dir_list = []
print('\n' + 'Renaming files present at:')
for root, dirs, files in os.walk(approot):
for file_list in content:
for filename in fnmatch.filter(files, file_list):
print(os.path.join(root, filename))
dir_list.append(root)
current_file = os.path.join(root, filename)
backup_file = os.path.join(root, filename + suffix)
os.rename(current_file, backup_file)
print("\n" + "Backup of all files complete!")
print('Backup of ' + str(len(dir_list)) + ' files taken recursively')
print('Number of files mentioned in text file: ' + str(len(content)) + '\n')
# 2 instances in UAT
# 12 instances in PROD
if (2*len(content)) == len(dir_list):
print("Retrofitted components will be copied to their respective directories")
for dst_ind in range(0, len(dir_list)):
if filename in fnmatch.filter(files, file_list):
print(source_dir + content[dst_ind] + "\t" + dir_list[dst_ind])
#shutil.copy2(source_dir+content[dst_ind], dir_list[dst_ind])
I'm getting the below error while copying the files (4.)
File contents:
['file1.class', 'file2.html']
Renaming files present at:
/home/saurabh/testbed/dest_dir/app_dir_1/file1.class
/home/saurabh/testbed/dest_dir/app_dir_1/file2.html
/home/saurabh/testbed/dest_dir/app_dir_2/file1.class
/home/saurabh/testbed/dest_dir/app_dir_2/file2.html
Backup of all files complete!
Backup of 4 files taken recursively
Number of files mentioned in text file: 2
Retrofitted components will be copied to their respective directories
/home/saurabh/testbed/patch_dir/file1.class /home/saurabh/testbed/dest_dir/app_dir_1
/home/saurabh/testbed/patch_dir/file2.html /home/saurabh/testbed/dest_dir/app_dir_1
Traceback (most recent call last):
File "./prod_movement.py", line 56, in <module>
print(source_dir + content[dst_ind] + "\t" + dir_list[dst_ind])
IndexError: list index out of range
Expected Output:
File contents:
['file1.class', 'file2.html']
Renaming files present at:
/home/saurabh/testbed/dest_dir/app_dir_1/file1.class
/home/saurabh/testbed/dest_dir/app_dir_1/file2.html
/home/saurabh/testbed/dest_dir/app_dir_2/file1.class
/home/saurabh/testbed/dest_dir/app_dir_2/file2.html
Backup of all files complete!
Backup of 4 files taken recursively
Number of files mentioned in text file: 2
Retrofitted components will be copied to their respective directories
/home/saurabh/testbed/patch_dir/file1.class /home/saurabh/testbed/dest_dir/app_dir_1
/home/saurabh/testbed/patch_dir/file2.html /home/saurabh/testbed/dest_dir/app_dir_1
/home/saurabh/testbed/patch_dir/file1.class /home/saurabh/testbed/dest_dir/app_dir_2
/home/saurabh/testbed/patch_dir/file2.html /home/saurabh/testbed/dest_dir/app_dir_2
Appreciate any help to fix the code.

Figured it out !!
#!/usr/bin/env python
import os
import fnmatch
import datetime
import shutil
with open('filenames.txt') as f:
content = f.readlines()
content = [x.strip() for x in content]
print('File contents:')
print(content)
suffix = datetime.datetime.now().strftime("_%d%b%Y")
approot = '/Users/saurabhm/Desktop/Python/testbed/dest_dir/'
source_dir = '/Users/saurabhm/Desktop/Python/testbed/patch_dir/'
dir_list = []
print('\n' + 'Renaming files present at:')
for root, dirs, files in os.walk(approot):
for file_list in content:
for filename in fnmatch.filter(files, file_list):
print(os.path.join(root, filename))
dir_list.append(root)
current_file = os.path.join(root, filename)
backup_file = os.path.join(root, filename + suffix)
#print(current_file, backup_file)
os.rename(current_file, backup_file)
#print(source_dir + filename + "\t" + root)
shutil.copy2(source_dir + filename, root)
os.chmod(root + '/' + filename, 0o750)
print("\n" + "Backup of all files complete!")
print('Backup of ' + str(len(dir_list)) + ' files taken recursively')
print('Number of files mentioned in text file: ' + str(len(content)) + '\n')
Once patched components are received over email, they are copied to a directory in the respective server. All these files are consolidated in a single directory (patch_dir)
Backup of existing files (having the same name and are present in "dest_dir") are taken wherever they are found, following which each file is copied from "patch_dir" to the directories inside "dest_dir", where their backup was taken.

How would I exclude directories from os.listdir results?

I'm making a script that will encode files within a directory using b64/b16 and I'm using os.listdir to do so, but it also lists directories which causes problems since now it's trying to encode directories as if it were a file.
How would I be able to exclude directories from os.listdir results?
import os
import sys
import base64
import codecs
import time
import string
import glob
#C:\\Users\\Fedora\\Desktop\\Win 10
path = "C:\\Users\\Fedora\\Desktop\\Win 10"
dirs = os.listdir(path)
files = []
filecount = 0
fileprogress = 0
for file in dirs:
files.append(file)
filecount = filecount + 1
for x in files:
os.system("cls")
fileprogress = fileprogress + 1
print("File " + str(fileprogress) + "/" + str(filecount))
print("Encrypting " + x + "...")
inputfile = open(path + "\\" + x, "rb")
data = inputfile.read()
inputfile.close()
data = base64.b16encode(data)
data = base64.b64encode(data)
data = base64.b16encode(data)
data = base64.b64encode(data)
data = base64.b16encode(data)
outputfile = open(path + "\\" + x + ".crypt", "wb")
outputfile.write(data)
outputfile.close()

use filter
filepath = "C:\\Users\\Fedora\\Desktop\\Win 10"
dirs = os.listdir(path)
files = filter(lambda x:os.path.isfile(os.path.join(filepath, x)), dirs)
or list comprehension with os.path.isfile()
filepath = "C:\\Users\\Fedora\\Desktop\\Win 10"
dirs = os.listdir(path)
files = [x for x in dirs if os.path.isfile(os.path.join(filepath, x))]

You can use os.path.isdir function to check if the current file is a directory.
Also, it is much better to use string formatting operations instead of string concatenation: not
print("File " + str(fileprogress) + "/" + str(filecount))
but
print("File {}/{}".format(fileprogress, filecount))
Such code is much easier to understand and modify.

Instead of using os.listdir() your can use os.walk which will return separate list for files and directories
python-oswalk-example
import os
path = "C:\\Users\\Fedora\\Desktop\\Win 10"
for (path, dirs, files) in os.walk(path):
print path
print dirs
print files
pythoncentral os-walk
#Import the os module, for the os.walk function
import os
#Set the directory you want to start from
path = "C:\\Users\\Fedora\\Desktop\\Win 10"
for dirName, subdirList, fileList in os.walk(path):
print('Found directory: %s' % dirName)
for fname in fileList:
print('\t%s' % fname)

Flatten complex directory structure in Python

I want to move files from a complex directory structure to just one place. For example i have this deep hierarchy:
foo/
foo2/
1.jpg
2.jpg
...
I want it to be:
1.jpg
2.jpg
...
My current solution:
def move(destination):
for_removal = os.path.join(destination, '\\')
is_in_parent = lambda x: x.find(for_removal) > -1
with directory(destination):
files_to_move = filter(is_in_parent,
glob_recursive(path='.'))
for file in files_to_move:
shutil.move(file, destination)
Definitions: directory and glob_recursive. Note, that my code only moves files to their common parent directory, not an arbitrary destination.
How can i move all files from a complex hierarchy to a single place succinctly and elegantly?

I don't like testing the name of the file about to be moved to see if we're already in the destination directory. Instead, this solution only scans the subdirectories of the destination
import os
import itertools
import shutil
def move(destination):
all_files = []
for root, _dirs, files in itertools.islice(os.walk(destination), 1, None):
for filename in files:
all_files.append(os.path.join(root, filename))
for filename in all_files:
shutil.move(filename, destination)
Explanation: os.walk walks recursively the destination in a "top down" manner. whole filenames are constructed with the os.path.join(root, filename) call. Now, to prevent scanning files at the top of the destination, we just need to ignore the first element of the iteration of os.walk. To do that I use islice(iterator, 1, None). One other more explicit way would be to do this:
def move(destination):
all_files = []
first_loop_pass = True
for root, _dirs, files in os.walk(destination):
if first_loop_pass:
first_loop_pass = False
continue
for filename in files:
all_files.append(os.path.join(root, filename))
for filename in all_files:
shutil.move(filename, destination)

this would do, it also renames files if they collide (I commented out the actual move and replaced with a copy):
import os
import sys
import string
import shutil
#Generate the file paths to traverse, or a single path if a file name was given
def getfiles(path):
if os.path.isdir(path):
for root, dirs, files in os.walk(path):
for name in files:
yield os.path.join(root, name)
else:
yield path
destination = "./newdir/"
fromdir = "./test/"
for f in getfiles(fromdir):
filename = string.split(f, '/')[-1]
if os.path.isfile(destination+filename):
filename = f.replace(fromdir,"",1).replace("/","_")
#os.rename(f, destination+filename)
shutil.copy(f, destination+filename)

Run recursively through directory, move the files and launch move for directories:
import shutil
import os
def move(destination, depth=None):
if not depth:
depth = []
for file_or_dir in os.listdir(os.path.join([destination] + depth, os.sep)):
if os.path.isfile(file_or_dir):
shutil.move(file_or_dir, destination)
else:
move(destination, os.path.join(depth + [file_or_dir], os.sep))

import os.path, shutil
def move(src, dest):
not_in_dest = lambda x: os.path.samefile(x, dest)
files_to_move = filter(not_in_dest,
glob_recursive(path=src))
for f in files_to_move:
shutil.move(f, dest)
Source for glob_recursive. Does not change name of file, if they collide.
samefile is a safe way to compare paths. But it doesn't work on Windows, so check How to emulate os.path.samefile behaviour on Windows and Python 2.7?.

def splitPath(p):
a,b = os.path.split(p)
return (splitPath(a) if len(a) and len(b) else []) + [b]
def safeprint(s):
try:
print(s)
except UnicodeEncodeError:
if sys.version_info >= (3,):
print(s.encode('utf8').decode(sys.stdout.encoding))
else:
print(s.encode('utf8'))
def flatten(root, doit):
SEP = "¦"
REPL = "?"
folderCount = 0
fileCount = 0
if not doit:
print("Simulating:")
for path, dirs, files in os.walk(root, topdown=False):
if path != root:
for f in files:
sp = splitPath(path)
np = ""
for element in sp[1:]:
e2 = element.replace(SEP, REPL)
np += e2 + SEP
f2 = f.replace(SEP, REPL)
newName = np + f2
safeprint("Moved: "+ newName )
if doit:
shutil.move(os.path.join(path, f), os.path.join(root, f))
# Uncomment, if you want filenames to be based on folder hierarchy.
#shutil.move(os.path.join(path, f), os.path.join(root, newName))
fileCount += 1
safeprint("Removed: "+ path)
if doit:
os.rmdir(path)
folderCount += 1
if doit:
print("Done.")
else:
print("Simulation complete.")
print("Moved files:", fileCount)
print("Removed folders:", folderCount)
directory_path = r"C:\Users\jd\Documents\myFtpData"
flatten(directory_path, True)

Adding on to the answers, I believe my answer will satisfy all your needs, the other answers fail when there is a subdirectory and file with the same filename as the upper directory.
This was SOLVED here, Also look at my Github Repo for Structured File Copy and Flattened File Copy:
import os, fnmatch, shutil
PATTERN = '*.txt' # Regex Pattern to Match files
INPUT_FOLDER = "A" # os.getcwd()
INPUT_FOLDER = os.path.abspath(INPUT_FOLDER)
include_input_foldername = False
prepend = "_included" if include_input_foldername else ""
OUTPUT_FOLDER = f"Structured_Copy_{os.path.basename(INPUT_FOLDER)}{prepend}"
os.makedirs(OUTPUT_FOLDER, exist_ok=True)
def find(pattern, path):
"""Utility to find files wrt a regex search"""
result = []
for root, dirs, files in os.walk(path):
for name in files:
if fnmatch.fnmatch(name, pattern):
result.append(os.path.join(root, name))
return result
all_files = find(PATTERN, INPUT_FOLDER)
for each_path in all_files:
relative_path = os.path.relpath(each_path, os.path.dirname(INPUT_FOLDER)) if include_input_foldername else os.path.relpath(each_path, INPUT_FOLDER)
flattened_relative_fullpath = os.path.join(OUTPUT_FOLDER, relative_path)
os.makedirs(os.path.dirname(flattened_relative_fullpath), exist_ok=True)
shutil.copy(each_path, flattened_relative_fullpath)
print(f"Copied {each_path} to {flattened_relative_fullpath}")
print(f"Finished Copying {len(all_files)} Files from : {INPUT_FOLDER} to : {OUTPUT_FOLDER}")