Rename csv columns in a folder of multiple csv - python

I have about 200 CSV files in a folder and some of the columns are not named the same for instance there is 'App name' ,'App Name' and 'app name' so I want to rename the this kind of columns to a standard name like 'App Name' and concatenate all the CSV's to one CSV file,

That would work:
import glob
import os
import pandas as pd
folder = 'the folder path'
filenames = [i for i in glob.glob(folder + os.path.sep + '*.csv')]
combined_csv = pd.concat([pd.read_csv(f, skiprows=1) for f in filenames])
combined_csv.columns = ["all the header labels"]

import glob
import pandas as pd
csv_folder = '/Users/yourname/folder_containing_csv/*.csv'
csv_file_list = []
for csv_path in glob.glob(csv_folder):
csv_file_list.append(csv_path)
for i in range(len(csv_file_list)):
df = pd.read_csv(csv_file_list[i], index_col=0)
df = df.rename(columns={"old_name": "new_name"})
file_name = str(csv_file_list[i]).split('.csv')[0].split('/')[-1:][0] # get file name by subsetting the path
df.to_csv('/Users/yourname/%(file_name)s.csv'%{'file_name': file_name}) # save to .csv using the same file name

Related

read in csv files from a folder and create html files

I'm new to python and hoping for some help to read in csv files from a folder and converting each file to a html folder...this is what I have so far:
import pandas as pd
import os
import glob
path = "htmlplots"
csv_files = glob.glob(os.path.join(path, "*.csv"))
for file in csv_files:
# read the csv file
df = pd.read_csv(file)
# print the filename
print('File Name:', file.split("\\")[-1])
# print the content
display(df)
Ideally I then need to create html files from the resulting csv files that have a 'next' and 'previous' link from one to two, two to three (next) and three to two, two to one (previous).
Use:
import pandas as pd
import os
import glob
path = ""
csv_files = glob.glob(os.path.join(path, "*.csv"))
for i, file in enumerate(csv_files):
df = pd.read_csv(file, header = None)
name = file.split('.')[-1]
if i>0:
prev = csv_files[i-1]
df.loc['prev',:]=f'http://{prev}'
else:
df.loc['prev',:]=''
if i!=len(csv_files)-1:
next = csv_files[i+1]
df.loc['next',:]=f'http://{next}'
else:
df.loc['next',:]=''
df.to_html(f"{file}.html", render_links = True)
Input csv file:
Output html:

How to call multiple columns from multiple csv file in python?

I have 100 csv file. I want to print particular columns from all the csv file with the file name. Here in this code I can print all of the csv file.
path = r'F:\11 semister\TPC_MEMBER'
all_files = glob.glob(path + "/*.csv")
dataStorage = {}
for filename in all_files:
name = os.path.basename(filename).split(".csv")[0]
dataStorage[name] = pd.read_csv(filename)
print(name)
dataStorage
May be you want this.
import pandas as pd
import numpy as np
import glob
path = r'folderpath' #provide your folder path where your csv files are stored.
all_csv= glob.glob(path + "/*.csv")
li = []
for filename in all_csv:
df = pd.read_csv(filename, index_col=None, header=0)
li.append(df)
data_frame = pd.concat(li, axis=0, ignore_index=True)
data_frame['columnname'] # enter the name of your dataframe's column.
print(data_frame)

selecting or matching specific columns by name lists[], or comma separate

I have a directory; inside this directory there are many sub-folders into it different extension files, so I just want to read only xlsx files, then I want to match columns by specifying the name. like using list [], or comma separate conditions . then I want to get the contents from the columns I have specified. some of the files has only one sheet and some has two sheets. the names in columns are the same for all the files. but the contents are different. so, please I need help to let the code go though files and search for only columns name that should be specified in the code, then combine them in one xlsx file output.xlsx
import pandas as pd
import numpy as np
import os
import xlrd
from xlrd import open_workbook
import sys
import xlsxwriter as xlsw
#import shutil
dataframes = []
path = r'C:\Users\malotaibi\Documents\mofaq', for root, subdirs, files in os.walk(path):,for file in files:, if file.endswith(".xlsx"):,f = (os.path.join(root, file)),print(f),dataframes.append(f),all_data = pd.DataFrame (),for f in dataframes:,dataframes = pd.read_excel(f, sheet_name=None) ,print(dataframes),list = dataframes[['Id', 'Original id', 'Name', 'Logic/Query', 'Comments']], collection= [list],join = pd.concat(collection),join.to_excel("output.xlsx")
Traceback (most recent call last): File
"C:\Users\malotaibi\Desktop\dd.py", line 39, in
list = dataframes[['Id', 'Original id', 'Name', 'Logic/Query', 'Comments']] TypeError: unhashable type: 'list'
your code might be:
import pandas as pd
import numpy as np
import os
import xlrd
from xlrd import open_workbook
import sys
import xlsxwriter as xlsw
#import shutil
dataframes = []
path = r'C:\Users\malotaibi\Documents\mofaq'
for root, subdirs, files in os.walk(path):
for file in files:
if file.endswith(".xlsx"):
f = (os.path.join(root, file))
print(f)
dataframes.append(f)
all_data = pd.DataFrame ()
for f in dataframes:
dataframes = pd.read_excel(f, sheet_name=None)
print(dataframes)
list = dataframes[['Id', 'Original id', 'Name', 'Logic/Query', 'Comments']]
collection= [list]
join = pd.concat(collection)
join.to_excel("output.xlsx")
try this:
import pandas as pd
import numpy as np
import os
import xlrd
from xlrd import open_workbook
import sys
import xlsxwriter as xlsw
#import shutil
dataframes = []
path = r'C:\Users\jainil\Documents'
for root, subdirs, files in os.walk(path):
for file in files:
if file.endswith(".xlsx"):
f = (os.path.join(root, file))
print(f)
dataframes.append(f)
all_data = pd.DataFrame ()
for f in dataframes:
dataframes = pd.read_excel(f, sheet_name=None)
print(dataframes)
list = dataframes['Sheet1']
list=list['Id', 'Original id', 'Name', 'Logic/Query', 'Comments']
collection= [list]
join = pd.concat(collection)
join.to_excel("output.xlsx")
print(join)
so you can try:
import pandas as pd
import numpy as np
import os
import xlrd
from xlrd import open_workbook
import sys
import xlsxwriter as xlsw
#import shutil
dataframes = []
path = r'C:\Users\jainil\Documents'
for root, subdirs, files in os.walk(path):
for file in files:
if file.endswith(".xlsx"):
f = (os.path.join(root, file))
print(f)
dataframes.append(f)
all_data = pd.DataFrame ()
join=pd.DataFrame(columns=['Id', 'Original id', 'Name', 'Logic/Query', 'Comments'])
for f in dataframes:
dataframes = pd.read_excel(f, sheet_name=None)
print(dataframes)
list = dataframes['Sheet1']
list=list['Id', 'Original id', 'Name', 'Logic/Query', 'Comments']
collection= [list]
join = pd.concat(join,collection)
join.to_excel("output.xlsx")
print(join)

Renaming files based on Dataframe content with Python and Pandas

I am trying to read a xlsx file, compare all the reference numbers from a column to files inside a folder and if they correspond, rename them to an email associate with the reference number.
Excel File has fields such as:
Reference EmailAddress
1123 bob.smith#yahoo.com
1233 john.drako#gmail.com
1334 samuel.manuel#yahoo.com
... .....
My folder applicants just contains doc files named as the Reference column:
How can I compare the contents of the applicantsCVs folder, to the Reference field inside my excel file and if it matches, rename all of the files as the corresponding email address ?
Here is What I've tried so far:
import os
import pandas as pd
dfOne = pd.read_excel('Book2.xlsx', na_values=['NA'], usecols = "A:D")
references = dfOne['Reference']
emailAddress = dfOne['EmailAddress']
cleanedEmailList = [x for x in emailAddress if str(x) != 'nan']
print(cleanedEmailList)
excelArray = []
filesArray = []
for root, dirs, files in os.walk("applicantCVs"):
for filename in files:
print(filename) #Original file name with type 1233.doc
reworkedFile = os.path.splitext(filename)[0]
filesArray.append(reworkedFile)
for entry in references:
excelArray.append(str(entry))
for i in excelArray:
if i in filesArray:
print(i, "corresponds to the file names")
I compare the reference names to the folder contents and print it out if it's the same:
for i in excelArray:
if i in filesArray:
print(i, "corresponds to the file names")
I've tried to rename it with os.rename(filename, cleanedEmailList ) but it didn't work because cleanedEmailList is an array of emails.
How can I match and rename the files?
Update:
from os.path import dirname
import pandas as pd
from pathlib import Path
import os
dfOne = pd.read_excel('Book2.xlsx', na_values=['NA'], usecols = "A:D")
emailAddress = dfOne['EmailAddress']
reference = dfOne['Reference'] = dfOne.references.astype(str)
references = dict(dfOne.dropna(subset=[reference, "EmailAddress"]).set_index(reference)["EmailAddress"])
print(references)
files = Path("applicantCVs").glob("*")
for file in files:
new_name = references.get(file.stem, file.stem)
file.rename(file.with_name(f"{new_name}{file.suffix}"))
based on sample data:
Reference EmailAddress
1123 bob.smith#yahoo.com
1233 john.drako#gmail.com
nan jane.smith#example.com
1334 samuel.manuel#yahoo.com
First you assemble a dict with the set of references as keys and the new names as values:
references = dict(df.dropna(subset=["Reference","EmailAddress"]).set_index("Reference")["EmailAddress"])
{'1123': 'bob.smith#yahoo.com',
'1233': 'john.drako#gmail.com',
'1334': 'samuel.manuel#yahoo.com'}
Note that the references are strs here. If they aren't in your original database, you can use astype(str)
Then you use pathlib.Path to look for all the files in the data directory:
files = Path("../data/renames").glob("*")
[WindowsPath('../data/renames/1123.docx'),
WindowsPath('../data/renames/1156.pptx'),
WindowsPath('../data/renames/1233.txt')]
The renaming can be made very simple:
for file in files:
new_name = references.get(file.stem, file.stem )
file.rename(file.with_name(f"{new_name}{file.suffix}"))
The references.get asks for the new filename, and if it doesn't find it, use the original stem.
[WindowsPath('../data/renames/1156.pptx'),
WindowsPath('../data/renames/bob.smith#yahoo.com.docx'),
WindowsPath('../data/renames/john.drako#gmail.com.txt')]
How about adding the "email associate" (your new name i guess?) into an dictionary, where the keys are your reference numbers?
This could look something like:
cor_dict = {}
for i in excelArray:
if i in filesArray:
cor_dict[i] =dfOne['EmailAddress'].at[dfOne.Reference == i]
for entry in cor_dict.items():
path = 'path to file...'
filename = str(entry[0])+'.doc'
new_filename = str(entry[1]).replace('#','_') + '_.doc'
filepath = os.path.join(path, filename)
new_filepath = os.path.join(path,new_filename)
os.rename(filename, new_filename)
This is one approach using a simple iteration.
Ex:
import os
#Sample Data#
#dfOne = pd.DataFrame({'Reference': [1123, 1233, 1334, 4444, 5555],'EmailAddress': ["bob.smith#yahoo.com", "john.drako#gmail.com", "samuel.manuel#yahoo.com", np.nan, "samuel.manuel#yahoo.com"]})
dfOne = pd.read_excel('Book2.xlsx', na_values=['NA'], usecols = "A:D")
dfOne.dropna(inplace=True) #Drop rows with NaN
for root, dirs, files in os.walk("applicantsCVs"):
for file in files:
file_name, ext = os.path.splitext(file)
email = dfOne[dfOne['Reference'].astype(str).str.contains(file_name)]["EmailAddress"]
if email.values:
os.rename(os.path.join(root, file), os.path.join(root, email.values[0]+ext))
Or if you have only .docx file to rename
import os
dfOne = pd.read_excel('Book2.xlsx', na_values=['NA'], usecols = "A:D")
dfOne["Reference"] = dfOne["Reference"].astype(str)
dfOne.dropna(inplace=True) #Drop rows with NaN
ext = ".docx"
for root, dirs, files in os.walk("applicantsCVs"):
files = r"\b" + "|".join(os.path.splitext(i)[0] for i in files) + r"\b"
for email, ref in dfOne[dfOne['Reference'].astype(str).str.contains(files, regex=True)].values:
os.rename(os.path.join(root, ref+ext), os.path.join(root, email+ext))
You could do it directly in your dataframe using df.apply():
import glob
import os.path
#Filter out null addresses
df = df.dropna(subset=['EmailAddress'])
#Add a column to check if file exists
df2['Existing_file'] = df2.apply(lambda row: glob.glob("applicantsCVs/{}.*".format(row['Reference'])), axis=1)
df2.apply(lambda row: os.rename(row.Existing_file[0], 'applicantsCVs/{}.{}'.format( row.EmailAddress, row.Existing_file[0].split('.')[-1])) if len(row.Existing_file) else None, axis = 1)
print(df2.Existing_file.map(len), "existing files renamed")
EDIT :
works now with any extension (.doc, .docx) by using glob module
Let consider our sample data in excel sheet is following:
Reference EmailAddress
1123 bob.smith#yahoo.com
1233 john.drako#gmail.com
1334 samuel.manuel#yahoo.com
nan python#gmail.com
There are following steps involved to solve this problem.
Step 1
import the data properly from excel sheet "my.xlsx". Here I am using the sample data
import pandas as pd
import os
#import data from excel sheet and drop rows with nan
df = pd.read_excel('my.xlsx').dropna()
#check the head of data if the data is in desirable format
df.head()
You will see that the data type in the references are in float type here
Step 2
Change the data type in the reference column to integer and then into string
df['Reference']=df.Reference.astype(int, inplace=True)
df = df.astype(str,inplace=True)
df.head()
Now the data is in desirable format
Step 3
Renaming the files in the desired folder. Zip the lists of 'Reference' and 'EmailAddress' to use in for loop.
#absolute path to folder. I consider you have the folder "application cv" in the home directory
path_to_files='/home/applicant cv/'
for ref,email in zip(list(df['Reference']),list(df['EmailAddress'])):
try:
os.rename(path_to_files+ref+'.doc',path_to_files+email+'.doc')
except:
print ("File name doesn't exist in the list, I am leaving it as it is")
Step 1: import the data from excel sheet "Book1.xlsx"
import pandas as pd
df = pd.read_excel (r'path of your file here\Book1.xlsx')
print (df)
Step 2: Choose path that your ".docx" files are in and store their names.
Get only relevent part of filename to compare.
mypath = r'path of docx files\doc files'
from os import listdir,rename
from os.path import isfile, join
onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))]
#print(onlyfiles)
currentfilename=onlyfiles[0].split(".")[0]
This is how I stored the files
Step 3: Run loop to check if name matches with the Reference. And just use rename(src,dest) function from os
for i in range(3):
#print(currentfilename,df['ref'][i])
if str(currentfilename)==str(df['Reference'][i]):
corrosponding_email=df['EmailAddress'][i]
#print(mypath+"\\"+corrosponding_email)
rename(mypath+"\\"+str(currentfilename)+".docx",mypath+"\\"+corrosponding_email+".docx")
checkout the code with example:https://github.com/Vineet-Dhaimodker

Python: Collect all xlsx. - Open - Add Column - Save in new folder

I am new with Python but trying to write a code which add a column on multiple .xlsx files and saves this files with the origin name to a new folder.
I have started with some coding beneath, but missing some code in open all files and saving to my DestPath. Would be pleased if any has a solution for this:
from os import listdir, path
import pandas as pd
import xlrd
SourcePath = 'C:\' #Source Path
DestPath = 'C:\' #Destination Path
# Listing up all .xlsx files from Source
def find_xlsx_filenames( path_to_dir, suffix=".xlsx" ):
filenames = listdir(path_to_dir)
return [ filename for filename in filenames if filename.endswith( suffix ) ]
filenames = find_xlsx_filenames(SourcePath)
fname = path.join(SourcePath, filenames[0]) # Tar første fil i mappa.
outname = path.join(outputdata, filenames[0])
for i in range(len(filenames)):
fname = path.join(SourcePath, filenames[i])
df = pd.read_excel(fname) #Read Excel file as a DataFrame
df['new_col'] = 'Sort Data' #Adding a new column named <Sort Data>
#To save it back as Excel
df.to_excel(DestPath, outname) #Write DateFrame back as Excel file
Thanks in Advance
check if this works
import os
import pandas as pd
path = 'C:/'
for roots, dirs, files in os.walk(path):
xlsfile = [ _ for _ in files if _.endswith('.xlsx')]
for xlsf in xlsfile:
df = pd.read_excel(os.path.join(roots, xlsf))
df['Sort Data'] = ' '
df.to_excel(os.path.join(roots, xlsf), index = False)

Categories

Resources