Develop Reference

Combing multiple csv files from multiple subfolders in one folder

Im trying to combine multiple files located in a directory. Each of the the files is located 3 subfolders(each subfolder has another folder or file) down from the main folder and I am unable to combine all of them. The best I can do is combine the ones in each bottom most subfolder. I can get a list of every specific file I want to combine from scanning but I can't combine them. I've gone through several methods and tutorials and can't find a way to do this. The code I have is below:
import pandas as pd
import os
import glob
os.getcwd()
path_of_the_directory = 'C:\\Users\\user\\Downloads\\top_folder'
ext = ('.csv')
for files in os.listdir(path_of_the_directory):
if files.endswith(ext):
print(files)
else:
continue
def list_files(dir):
r = []
for root, dirs, files in os.walk(dir):
for name in files:
filepath = root + os.sep + name
if filepath.endswith(".csv"):
r.append(os.path.join(root, name))
return r
print(r)
files = []
for file in r:
#for dir, dir_name, file_list in os.walk(path):
files.append(os.path.join(path,file))
combined_df = pd.concat([pd.read_csv(file) for file in files])
df = pd.concat([pd.read_csv(f) for f in files])
df.to_csv("merged.csv")
print(files)
list_files(data_dir)
data_dir = r'C:\\Users\\user\\Downloads\top_folder'
sub_folders = os.listdir(data_dir)
sub_folders
path = os.path.join(data_dir, sub_folders[2])
os.chdir(path)
files = glob.glob(path + ".\*\*.csv")
files
df = pd.concat([pd.read_csv(f) for f in chat_files])
df.to_csv("merged.csv")
Any help or direction would be extremely appreciated.

Show path tree with files

import os
path = "G:\krunker\mod"
abcde = open("path.txt", "w")
for dirpath, dirnames, filenames in os.walk(path):
directory_level = dirpath.replace(path, "")
directory_level = directory_level.count(os.sep)
indent = " " * 4
print("{}{}/".format(indent*directory_level, os.path.basename(dirpath)), file=abcde)
for f in filenames:
print("{}{}".format(indent*(directory_level+1), f), file=abcde)
abcde.close()
I want it to print the files in every single folder of the path but it does only on the last

The indentation is not correct. The second for loop also has to be inside the first for loop.
Correct code:
import os
path = "/home/user/my_folder/tools"
abcde = open("path.txt", "w")
for dirpath, dirnames, filenames in os.walk(path):
directory_level = dirpath.replace(path, "")
directory_level = directory_level.count(os.sep)
indent = " " * 4
print("{}{}/".format(indent*directory_level, os.path.basename(dirpath)), file=abcde)
for f in filenames:
print("{}{}".format(indent*(directory_level+1), f), file=abcde)
abcde.close()
A part of path.txt content:
tools/
.gitignore
README.md
__init__.py
requirements3.txt
test.py
path.txt
.git/
description
hooks/
commit-msg.sample
info/
exclude
refs/
heads/
master

You can also use a recursive function which gets recursivly all subfolder's content until there's any subfolder:
from os import walk
output = open("path.txt", "w")
def listFiles(path, indent):
for (openedPath, folders, files) in walk(path):
for file in files:
output.write("\t" * (indent) + file + "\n")
for folder in folders:
output.write("\t" * (indent) + folder + "/\n")
listFiles(path + "/" + folder, indent + 1)
break
source = "/my/path/to/my/folder"
print(source + "/")
listFiles(source, 1)
There's an example with a little code project folder.
/my/path/to/my/folder/
input.txt
main.py
output/
error.cpp
trying.cpp
logo.cpp

You can use this kind of code to simplify
import os
folder = r"C:\path\to\find\files"
x = [os.path.join(r,file) for r,d,f in os.walk(folder) for file in f # If you want specific files if file.endswith(".txt")]
y = [os.path.join(r,folder) for r,d,f in os.walk(folder) for folder in d]
print(x) #For files in main directory and subdirectories
print(y) #For files in main directory and subdirectories

Copy files from multiple specific subfolders

Under the file path D:/src, I have images folders and its subfolders which have a regular structure as follows:
Folder A
- Subfolder a
- Subfolder b
- Subfolder c
Folder B
- Subfolder a
- Subfolder b
- Subfolder c
- Subfolder d
Folder C
- Subfolder a
- Subfolder b
- Subfolder c
...
I want to copy all .jpg files in Subfolder b from Folder A, B, C and so on to a new folder Subfolder b in D:/dst. How can I do it in Python? Thanks.
Subfolder b
-xxx.jpg
-xyx.jpg
-yxz.jpg
...
Here is what I have found from the following link may helps:
Copy certain files from one folder to another using python
import os;
import shutil;
import glob;
source="C:/Users/X/Pictures/test/Z.jpg"
dest="C:/Users/Public/Image"
if os.path.exists(dest):
print("this folder exit in this dir")
else:
dir = os.mkdir(dest)
for file in glob._iglob(os.path.join(source),""):
shutil.copy(file,dest)
print("done")

Try this
import os
from os.path import join, isfile
BASE_PATH = 'd:/test'
SUBFOLDER = 'Subfolder b'
for folder, subfolders, *_ in os.walk(BASE_PATH):
if SUBFOLDER in subfolders:
full_path = join(BASE_PATH, folder, SUBFOLDER)
files = [f for f in os.listdir(full_path) if isfile(join(full_path, f)) and f.lower().endswith(('.jpg', '.jpeg'))]
for f in files:
file_path = join(full_path, f)
print (f'Copy {f} somewehere')

Assuming you have 2 levels of nesting
root_dir = './data'
dest_dir = './new_location'
os.listdir(root_dir)
for folder in os.listdir(root_dir):
folder_path = os.path.join(root_dir, folder)
if os.path.isdir(folder_path):
for subfolder in os.listdir(folder_path):
if subfolder == 'Subfolder B':
subfolder_path = os.path.join(root_dir, folder, subfolder)
print(subfolder_path)
for filename in os.listdir(subfolder_path):
file_path = os.path.join(root_dir, folder, subfolder, filename)
dest_path = os.path.join(dest_dir, filename)
shutil.copy(file_path, dest_path)
print("Copied ", file_path, "to", dest_path)
You just need 2 for loops, and in the the inner for loop you just check whether the folder name matches Subfolder B. If it does then copy all the files inside that directory to your destination folder.

Here's a short script that should do the work...
import os
# list all the directories in current directory
dirs = [x[0] for x in os.walk("D:/src")]
for d in dirs:
## list all files in A/b/*, B/b/*, C/b/*...
files_to_copy = os.listdir(os.path.join(d, "b"))
for f in files_to_copy:
if f.endswith(".jpg"): ## copy the relevant files to dest
shutil.copy(os.path.join(d, "b", f), os.path.join(dest, f))

import shutil
import os
def move_files(source_director, destination_director):
# select the files in the source directory
fisiere = [fisier for fisier in os.listdir(source_director) if os.path.isfile(os.path.join(source_director, fisier))]
print(fisiere)
# each file in the source directory is moved to the destination directory
for fisier in fisiere:
shutil.move(source_director + '\\' + fisier, destination_director)
def copy_files(source_director, destination_director):
# select the files in the source directory
fisiere = [fisier for fisier in os.listdir(source_director) if os.path.isfile(os.path.join(source_director, fisier))]
print(fisiere)
# each file in the source directory is copied to the destination directory
for fisier in fisiere:
shutil.copy(source_director + '\\' + fisier, destination_director)
if __name__ == '__main__':
# WARNING: paths to directories must be written with '\\' (double backslash)
# here you set the DIN directory you want to move / copy the files
source_director = 'c:\\Folder'
destination_director = 'c:\\Folder\\Subfolder1\\Subfolder2'
#CASE 1
# move_files(source_director, destination_director)
#CASE 2
copy_files(source_director, destination_director)
THE SOURCE OF THIS CODE HERE:
https://neculaifantanaru.com/en/python-copy-or-moves-files-from-a-folder-to-another-folder.html

Getting paths of each file of a directory into an Array in python

Im trying to put into an array files[] the paths of each file from the Data folder but when I try to go into subfolders I want it to be able to go down to the end of the Data file, for example I can read files in a subfolder of the main folder Data which im trying to get a list of all the paths of each file into an array but it doesn't go deeper it does not access the subfolder of the subfolder of Data without writing a loop. Want I want is a loop which has infinit depth of view of files in the Data folder so I can get all the file paths.
For example this is what I get:
['Data/DataReader.py', 'Data/DataReader - Copy.py', 'Data/Dat/DataReader.py', 'Data/fge/er.txt']
This is what I want but it can still go into deeper folders:
['Data/DataReader.py', 'Data/DataReader - Copy.py', 'Data/Dat/DataReader.py', 'Data/fge/er.txt', 'Data/fge/Folder/dummy.png', 'Data/fge/Folder/AnotherFolder/data.dat']
This is my current path, what would i need to add or change?
import os
from os import walk
files = []
folders = []
for (dirname, dirpath, filename) in walk('Data'):
folders.extend(dirpath)
files.extend(filename)
break
filecount = 0
for i in files:
i = 'Data/' + i
files[filecount] = i
filecount += 1
foldercount = 0
for i in folders:
i = 'Data/' + i
folders[foldercount] = i
foldercount += 1
subfolders = []
subf_files = []
for i in folders:
for (dirname, dirpath, filename) in walk(i):
subfolders.extend(dirpath)
subf_files.extend(filename)
break
subf_files_count = 0
for a in subf_files:
a = i + '/'+a
files = files
files.append(a)
print files
subf_files = []
print files
print folders
Thanks a lot!

Don't understand what are your trying to do, especially why you break your walk after the first element:
import os
files = []
folders = []
for (path, dirnames, filenames) in os.walk('Data'):
folders.extend(os.path.join(path, name) for name in dirnames)
files.extend(os.path.join(path, name) for name in filenames)
print files
print folders

Flatten complex directory structure in Python

I want to move files from a complex directory structure to just one place. For example i have this deep hierarchy:
foo/
foo2/
1.jpg
2.jpg
...
I want it to be:
1.jpg
2.jpg
...
My current solution:
def move(destination):
for_removal = os.path.join(destination, '\\')
is_in_parent = lambda x: x.find(for_removal) > -1
with directory(destination):
files_to_move = filter(is_in_parent,
glob_recursive(path='.'))
for file in files_to_move:
shutil.move(file, destination)
Definitions: directory and glob_recursive. Note, that my code only moves files to their common parent directory, not an arbitrary destination.
How can i move all files from a complex hierarchy to a single place succinctly and elegantly?

I don't like testing the name of the file about to be moved to see if we're already in the destination directory. Instead, this solution only scans the subdirectories of the destination
import os
import itertools
import shutil
def move(destination):
all_files = []
for root, _dirs, files in itertools.islice(os.walk(destination), 1, None):
for filename in files:
all_files.append(os.path.join(root, filename))
for filename in all_files:
shutil.move(filename, destination)
Explanation: os.walk walks recursively the destination in a "top down" manner. whole filenames are constructed with the os.path.join(root, filename) call. Now, to prevent scanning files at the top of the destination, we just need to ignore the first element of the iteration of os.walk. To do that I use islice(iterator, 1, None). One other more explicit way would be to do this:
def move(destination):
all_files = []
first_loop_pass = True
for root, _dirs, files in os.walk(destination):
if first_loop_pass:
first_loop_pass = False
continue
for filename in files:
all_files.append(os.path.join(root, filename))
for filename in all_files:
shutil.move(filename, destination)

this would do, it also renames files if they collide (I commented out the actual move and replaced with a copy):
import os
import sys
import string
import shutil
#Generate the file paths to traverse, or a single path if a file name was given
def getfiles(path):
if os.path.isdir(path):
for root, dirs, files in os.walk(path):
for name in files:
yield os.path.join(root, name)
else:
yield path
destination = "./newdir/"
fromdir = "./test/"
for f in getfiles(fromdir):
filename = string.split(f, '/')[-1]
if os.path.isfile(destination+filename):
filename = f.replace(fromdir,"",1).replace("/","_")
#os.rename(f, destination+filename)
shutil.copy(f, destination+filename)

Run recursively through directory, move the files and launch move for directories:
import shutil
import os
def move(destination, depth=None):
if not depth:
depth = []
for file_or_dir in os.listdir(os.path.join([destination] + depth, os.sep)):
if os.path.isfile(file_or_dir):
shutil.move(file_or_dir, destination)
else:
move(destination, os.path.join(depth + [file_or_dir], os.sep))

import os.path, shutil
def move(src, dest):
not_in_dest = lambda x: os.path.samefile(x, dest)
files_to_move = filter(not_in_dest,
glob_recursive(path=src))
for f in files_to_move:
shutil.move(f, dest)
Source for glob_recursive. Does not change name of file, if they collide.
samefile is a safe way to compare paths. But it doesn't work on Windows, so check How to emulate os.path.samefile behaviour on Windows and Python 2.7?.

def splitPath(p):
a,b = os.path.split(p)
return (splitPath(a) if len(a) and len(b) else []) + [b]
def safeprint(s):
try:
print(s)
except UnicodeEncodeError:
if sys.version_info >= (3,):
print(s.encode('utf8').decode(sys.stdout.encoding))
else:
print(s.encode('utf8'))
def flatten(root, doit):
SEP = "¦"
REPL = "?"
folderCount = 0
fileCount = 0
if not doit:
print("Simulating:")
for path, dirs, files in os.walk(root, topdown=False):
if path != root:
for f in files:
sp = splitPath(path)
np = ""
for element in sp[1:]:
e2 = element.replace(SEP, REPL)
np += e2 + SEP
f2 = f.replace(SEP, REPL)
newName = np + f2
safeprint("Moved: "+ newName )
if doit:
shutil.move(os.path.join(path, f), os.path.join(root, f))
# Uncomment, if you want filenames to be based on folder hierarchy.
#shutil.move(os.path.join(path, f), os.path.join(root, newName))
fileCount += 1
safeprint("Removed: "+ path)
if doit:
os.rmdir(path)
folderCount += 1
if doit:
print("Done.")
else:
print("Simulation complete.")
print("Moved files:", fileCount)
print("Removed folders:", folderCount)
directory_path = r"C:\Users\jd\Documents\myFtpData"
flatten(directory_path, True)

Adding on to the answers, I believe my answer will satisfy all your needs, the other answers fail when there is a subdirectory and file with the same filename as the upper directory.
This was SOLVED here, Also look at my Github Repo for Structured File Copy and Flattened File Copy:
import os, fnmatch, shutil
PATTERN = '*.txt' # Regex Pattern to Match files
INPUT_FOLDER = "A" # os.getcwd()
INPUT_FOLDER = os.path.abspath(INPUT_FOLDER)
include_input_foldername = False
prepend = "_included" if include_input_foldername else ""
OUTPUT_FOLDER = f"Structured_Copy_{os.path.basename(INPUT_FOLDER)}{prepend}"
os.makedirs(OUTPUT_FOLDER, exist_ok=True)
def find(pattern, path):
"""Utility to find files wrt a regex search"""
result = []
for root, dirs, files in os.walk(path):
for name in files:
if fnmatch.fnmatch(name, pattern):
result.append(os.path.join(root, name))
return result
all_files = find(PATTERN, INPUT_FOLDER)
for each_path in all_files:
relative_path = os.path.relpath(each_path, os.path.dirname(INPUT_FOLDER)) if include_input_foldername else os.path.relpath(each_path, INPUT_FOLDER)
flattened_relative_fullpath = os.path.join(OUTPUT_FOLDER, relative_path)
os.makedirs(os.path.dirname(flattened_relative_fullpath), exist_ok=True)
shutil.copy(each_path, flattened_relative_fullpath)
print(f"Copied {each_path} to {flattened_relative_fullpath}")
print(f"Finished Copying {len(all_files)} Files from : {INPUT_FOLDER} to : {OUTPUT_FOLDER}")