So I am reading in a bunch of .xlsx files from a directory and I want to transform them into a table.
Easy, but I am running into the problem that these excel files do not have the same headers. How can I go about creating a code that will check the excel files headers, and either append it to a table that has the same columns, or create a new one if it that format of columns does not exist.
My Code:
import sqlite3 as sql
import pandas as pd
import os
def obtain_data(filename, db):
connect = sql.connect('filepath.sqlite3')
workbook = pd.ExcelFile('filepath' + filename)
df = workbook.parse('Sheet1')
new_db = db.append(df)
print(new_db)
new_db = new_db.rename(columns={'INDEX': 'INDX'})
connect.close()
return new_db
usable_files = []
for filename in os.listdir('filepath'):
if filename.endswith(".xlsx"):
print(filename)
usable_files.append(filename)
else:
print('no')
print(filename)
new_db = pd.DataFrame()
for file in usable_files:
new_db= new_db.append(obtain_data(file, new_db))
Note, I do not know in advanced whether the excel file will have a matching pair of columns or not. Thanks in advance.
UPDATE:
def obtain_data(filename, connect, data):
workbook = pd.ExcelFile('filpath' + filename)
df = workbook.parse('Sheet1')
df = df.rename(columns={'INDEX': 'INDX'})
headers = new_db.dtypes.index
header_list = str(headers.tolist())
header_list = ''.join(header_list)
hash_t = str(hashlib.md5(header_list.encode('utf-8')).hexdigest())
if hash_t not in hash_list:
x = pd.DataFrame(df)
print(x.name)
x.name = hash_t
print(x.name)
hash_list.append(hash_t)
data_frames = data.append(x)
connect.close()
elif hash_t in hash_list:
print('hash is repeating. Find a way to make this code get table name.')
print(filename + ' has been completed succesfully.')
final_results = {'df': df, 'hash_t': hash_t}
return final_results
usable_files = []
for filename in os.listdir('filepath'):
if filename.endswith(".xlsx"):
usable_files.append(filename)
else:
print('cool')
hash_list = []
data_frames = []
new_db = pd.DataFrame()
for file in usable_files:
connect = sql.connect('filepath_test.sqlite3')
x = new_db.append(obtain_data(file, connect, data_frames),
ignore_index=True)
if x['hash_t'] not in hash_list:
new_db = new_db.append(x['df'])
new_db.append(x['hash_t'])
else:
new_db = new_db.append(x['df'])
print(new_db)
connect.commit()
connect.close()
Not sure if this is exactly what you're looking for, but have a look. If your dataframes have common column names, they're merged together resulting in a new dataframe with all the columns in both dataframes, and any overlapping entry names are combined into a single row (I'm not sure if this is what you want) EDIT: For an example of this, see how the two Toms are combined in the output.
If the two dataframes don't have any columns in common, they're simply concatenated, resulting in a dataframe with the columns from both dataframes, but with no merging of overlapping entry names.
I've included a (pretty long) printout to make it clearer what's going on.
import pandas as pd
def merge_dataframes(merge_this_df, with_this_df):
print "-----------------------------------------------------"
print "Merging this:"
print merge_this_df
print "\nWith this:"
print with_this_df
print "\nResult:"
# Check if they have common columns
any_common_columns = any([column_name in merge_this_df.columns for column_name in with_this_df.columns])
if any_common_columns:
merged_df = merge_this_df.merge(with_this_df, how="outer")
print merged_df
return merged_df
else:
concatenated_df = pd.concat([merge_this_df, with_this_df])
print concatenated_df
return concatenated_df
# Create some dummy data
df = pd.DataFrame({
"name": ["Tom", "David", "Helen"],
"age": ["30", "40", "50"]
})
df2 = pd.DataFrame({
"name": ["Tom", "Juan", "Maria", "Julia"],
"occupation": ["Plumber", "Chef", "Astronaut", "Teacher"],
})
df3 = pd.DataFrame({
"animal": ["Cat", "Platypus"],
"food": ["Catfoot", "Platypus-food"]
})
# Collect all dummy data in a list
all_dfs = [df, df2, df3]
# Merge or concatenate all dataframes in to a single dataframe
final_df = pd.DataFrame()
for dataframe in all_dfs:
final_df = merge_dataframes(final_df, dataframe)
Printout:
-----------------------------------------------------
Merging this:
Empty DataFrame
Columns: []
Index: []
With this:
age name
0 30 Tom
1 40 David
2 50 Helen
Result:
age name
0 30 Tom
1 40 David
2 50 Helen
-----------------------------------------------------
Merging this:
age name
0 30 Tom
1 40 David
2 50 Helen
With this:
name occupation
0 Tom Plumber
1 Juan Chef
2 Maria Astronaut
3 Julia Teacher
Result:
age name occupation
0 30 Tom Plumber
1 40 David NaN
2 50 Helen NaN
3 NaN Juan Chef
4 NaN Maria Astronaut
5 NaN Julia Teacher
-----------------------------------------------------
Merging this:
age name occupation
0 30 Tom Plumber
1 40 David NaN
2 50 Helen NaN
3 NaN Juan Chef
4 NaN Maria Astronaut
5 NaN Julia Teacher
With this:
animal food
0 Cat Catfoot
1 Platypus Platypus-food
Result:
age animal food name occupation
0 30 NaN NaN Tom Plumber
1 40 NaN NaN David NaN
2 50 NaN NaN Helen NaN
3 NaN NaN NaN Juan Chef
4 NaN NaN NaN Maria Astronaut
5 NaN NaN NaN Julia Teacher
0 NaN Cat Catfoot NaN NaN
1 NaN Platypus Platypus-food NaN NaN
EDIT2: Another approach: Read sqlite database into pandas dataframe -> fix column-related stuff -> write pandas dataframe into sqlite database (overwriting the previous one):
import sqlite3 as sql
import pandas as pd
import os
def obtain_data(df_to_add):
# Connect to database
connect = sql.connect("my_database.sqlite")
print "--------------------------------------"
# Read current database into a dataframe
try:
current_df = pd.read_sql_query("SELECT * FROM my_database", connect)
print "Database currently looks like:"
print current_df
# Now, we check if we have overlapping column names in our database and our dataframe
if any([c in current_df.columns for c in df_to_add.columns]):
# If we do, we can merge them
new_df = current_df.merge(df_to_add, how="outer")
else:
# If there are no common columns, we just concatenate them
new_df = pd.concat([current_df, df_to_add])
# Now, we simply overwrite the DB with our current dataframe
print "Adding to database"
new_df.to_sql("my_database", connect, if_exists="replace", index=False)
# For good measure, read database again and print it out
database_df = pd.read_sql_query("SELECT * FROM my_database", connect)
print "Database now looks like:"
print database_df
connect.close()
except pd.io.sql.DatabaseError:
# There's no database called my_database, so simply insert our dataframe
print "Creating initial database named my_database"
df_to_add.to_sql("my_database", connect, index=False)
print "Current database:"
print df_to_add
# We're done here
connect.close()
return
# Create some dummy data
df1 = pd.DataFrame({
"name": ["Tom", "David", "Helen"],
"age": ["30", "40", "50"]
})
df2 = pd.DataFrame({
"name": ["Tom", "Juan", "Maria", "Julia"],
"occupation": ["Plumber", "Chef", "Astronaut", "Teacher"],
})
df3 = pd.DataFrame({
"animal": ["Cat", "Platypus"],
"food": ["Catfoot", "Platypus-food"]
})
# Read all dummy data into the database
for df in [df1, df2, df3]:
obtain_data(df)
And the output:
--------------------------------------
Creating initial database named my_database
Current database:
age name
0 30 Tom
1 40 David
2 50 Helen
--------------------------------------
Database currently looks like:
age name
0 30 Tom
1 40 David
2 50 Helen
Adding to database
Database now looks like:
age name occupation
0 30 Tom Plumber
1 40 David None
2 50 Helen None
3 None Juan Chef
4 None Maria Astronaut
5 None Julia Teacher
--------------------------------------
Database currently looks like:
age name occupation
0 30 Tom Plumber
1 40 David None
2 50 Helen None
3 None Juan Chef
4 None Maria Astronaut
5 None Julia Teacher
Adding to database
Database now looks like:
age animal food name occupation
0 30 None None Tom Plumber
1 40 None None David None
2 50 None None Helen None
3 None None None Juan Chef
4 None None None Maria Astronaut
5 None None None Julia Teacher
6 None Cat Catfoot None None
7 None Platypus Platypus-food None None
Let me know if this isn't what your're looking for.
Related
Lot of answers on merging and full col, but can't figure out a more effective method. for my situation.
current version of python, pandas, numpy, and file format is parquet
Simply put if col1 ==x the col 10 = 1, col11 = 2, col... etc.
look1 = 'EMPLOYEE'
look2 = 'CHESTER'
look3 = "TONY'S"
look4 = "VICTOR'S"
tgt1 = 'inv_group'
tgt2 = 'acc_num'
for x in range(len(df['ph_name'])):
df[tgt1][x] = 'MEMORIAL'
df[tgt2][x] = 12345
elif df['ph_name'][x] == look2:
df[tgt1][x] = 'WALMART'
df[tgt2][x] = 45678
elif df['ph_name'][x] == look3:
df[tgt1][x] = 'TONYS'
df[tgt2][x] = 27359
elif df['ph_name'][x] == look4:
df[tgt1][x] = 'VICTOR'
df[tgt2][x] = 45378
basic sample:
unit_name tgt1 tgt2
0 EMPLOYEE Nan Nan
1 EMPLOYEE Nan Nan
2 TONY'S Nan Nan
3 CHESTER Nan Nan
4 VICTOR'S Nan Nan
5 EMPLOYEE Nan Nan
GOAL:
unit_name tgt1 tgt2
0 EMPLOYEE MEMORIAL 12345
1 EMPLOYEE MEMORIAL 12345
2 TONY'S TONYS 27359
3 CHESTER WALMART 45678
4 VICTOR'S VICTOR 45378
5 EMPLOYEE MEMORIAL 12345
So this works... I get the custom columns values added, It's not the fastest under the sun, but it works.
It takes 6.2429744 on 28896 rows. I'm concerned when I put it to the grind, It's going to start dragging me down.
The other downside is I get this annoyance... Yes I can silence, but I feel like this might be due to a bad practice that I should know how to curtail.
SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame
Basically...
Is there a way to optimize this?
Is this warning due to a bad habit, my ignorance, or do I just need to silence it?
Given: (It's silly to have all NaN columns)
unit_name
0 EMPLOYEE
1 EMPLOYEE
2 TONY'S
3 CHESTER
4 VICTOR'S
5 EMPLOYEE
df = pd.DataFrame({'unit_name': {0: 'EMPLOYEE', 1: 'EMPLOYEE', 2: "TONY'S", 3: 'CHESTER', 4: "VICTOR'S", 5: 'EMPLOYEE'}})
Doing: (Let's use pd.Series.map and create a dictionary for easier future modification)
looks = ['EMPLOYEE', 'CHESTER', "TONY'S", "VICTOR'S"]
new_cols = {
'inv_group': ["MEMORIAL", "WALMART", "TONYS", "VICTOR"],
'acc_num': [12345, 45678, 27359, 45378]
}
for col, values in new_cols.items():
df[col] = df['unit_name'].map(dict(zip(looks, values)))
print(df)
Output: (I assumed you'd typed the column names wrong)
unit_name inv_group acc_num
0 EMPLOYEE MEMORIAL 12345
1 EMPLOYEE MEMORIAL 12345
2 TONY'S TONYS 27359
3 CHESTER WALMART 45678
4 VICTOR'S VICTOR 45378
5 EMPLOYEE MEMORIAL 12345
Flying blind here since I don't see your data:
cond_list = [df["ph_name"] == look for look in [look1, look2, look3, look4]]
# Rows ph_name outside of the list will keep their original values
df[tgt1] = np.select(cond_list, ["MEMORIAL", "WALMART", "TONY'S", "VICTOR"])
df[tgt2] = np.select(cond_list, [12345, 45678, 27359, 45378])
I have a big series of strings that I want to split into a dataframe.
The series looks like this:
s = pd.Series({"1":"Name=Marc-Age=48-Car=Tesla",
"2":"Name=Ben-Job=Pilot-Car=Porsche",
"3":"Name=Tom-Age=24-Car=Ford"})
I want to split this into a dataframe looking like this:
Name Age Job Car
1 Marc 48 Nan Tesla
2 Ben Nan Pilot Porsche
3 Tom 24 Nan Ford
I tried to split the strings first by "-" and then by "=" but don't understand how to continue after
df=s.str.split("-", expand=True)
for col in df.columns:
df[col]=df[col].str.split("=")
I get this:
` 0 1 2`
`1 ['Name', 'Marc'] ['Age', '48'] ['Car', 'Tesla']`
`2 ['Name', 'Ben'] ['Job', 'Pilot'] ['Car', 'Porsche']`
`3 ['Name', 'Tom'] ['Age', '24'] ['Car', 'Ford']`
I don't know how to continue from here. I can't loop through the rows because my dataset is really big.
Can anyone help on how to go on from here?
If you split then explode and split again you can then use a pivot.
import pandas as pd
s = pd.Series({"1":"Name=Marc-Age=48-Car=Tesla",
"2":"Name=Ben-Job=Pilot-Car=Porsche",
"3":"Name=Tom-Age=24-Car=Ford"})
s = s.str.split('-').explode().str.split('=', expand=True).reset_index()
s = s.pivot(index='index', columns=0, values=1).reset_index(drop=True)
Output
Age Car Job Name
0 48 Tesla NaN Marc
1 NaN Porsche Pilot Ben
2 24 Ford NaN Tom
I have a list of objects by each name and a dataframe like this.
Jimmy = ['chair','table','pencil']
Charles = ['smartphone','cake']
John = ['clock','paper']
id
name
1
Jimmy
2
Charles
3
John
I would like to use a loop that allows me to obtain the following result.
id
name
picks
1
Jimmy
chair
1
Jimmy
table
1
Jimmy
pencil
2
Charles
smartphone
2
Charles
cake
3
John
clock
3
John
paper
You can assign and explode:
values = {'Jimmy': Jimmy, 'Charles': Charles, 'John': John}
out = df.assign(picks=df['name'].map(values)).explode('picks')
Or set up a DataFrame, stack and merge:
values = {'Jimmy': Jimmy, 'Charles': Charles, 'John': John}
out = df.merge(
pd.DataFrame.from_dict(values, orient='index')
.stack().droplevel(1).rename('picks'),
left_on='name', right_index=True
)
output:
id name picks
0 1 Jimmy chair
0 1 Jimmy table
0 1 Jimmy pencil
1 2 Charles smartphone
1 2 Charles cake
2 3 John clock
2 3 John paper
We can make a dataframe relating names to picks, then join them together with merge:
import pandas as pd
#dataframe from question
df = pd.DataFrame()
df["id"] = [1, 2, 3]
df["name"] = ["Jimmy", "Charles", "John"]
#dataframe relating names to picks.
picks_df = pd.DataFrame()
picks_df["name"] = ["Jimmy", "Jimmy", "Jimmy", "Charles", "Charles", "John", "John"]
picks_df["picks"] = ["chair", "table", "pencil", "smartphone", "cake", "clock", "paper"]
#Merge and print
print(pd.merge(df, picks_df, on="name"))
I was comparing two excel files which contains the information of the students of two schools. However those files might contain different number of rows between them.
The first set that I used is to import the excel files in two dataframes:
df1 = pd.read_excel('School A - Information.xlsx')
df2 = pd.read_excel('School B - Information.xlsx')
print(df1)
Name Age Birth_Country Previous Schools
0 tom 10 USA 3
1 nick 15 MEX 1
2 juli 14 CAN 0
3 tom 19 NOR 1
print(df2)
Name Age Birth_Country Previous Schools
0 tom 10 USA 3
1 tom 19 NOR 1
2 nick 15 MEX 4
After this, I would like to check the divergences between those two dataframes (index order is not important). However I am receiving an error due to the size of the dataframes.
compare = df1.values == df2.values
<ipython-input-9-7cc64ba0e622>:1: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.
compare = df1.values == df2.values
print(compare)
False
Adding to that, I would like to create a third DataFrame with the corresponding divergences, that shows the divergence.
import numpy as np
rows,cols=np.where(compare==False)
for item in zip(rows,cols):
df1.iloc[item[0], item[1]] = '{} --> {}'.format(df1.iloc[item[0], item[1]],df2.iloc[item[0], item[1]])
However, using this code is not working, as the index order may be different between the two dataframes.
My expected output should be the below dataframe:
You can use pd.merge to accomplish this. If you're unfamiliar with dataframe merges, here's a post that describes relational database merging ideas: link. So in this case, what we want to do is first do a left merge of df2 onto df1 to find how the Previous Schools column differs:
df_merged = pd.merge(df1, df2, how="left", on=["Name", "Age", "Birth_Country"], suffixes=["_A", "_B"])
print(df_merged)
will give you a new dataframe
Name Age Birth_Country Previous Schools_A Previous Schools_B
0 tom 10 USA 3 3.0
1 nick 15 MEX 1 4.0
2 juli 14 CAN 0 NaN
3 tom 19 NOR 1 1.0
This new dataframe has all the information you're looking for. To find just the rows where the Previous Schools entries differ:
df_different = df_merged[df_merged["Previous Schools_A"]!=df_merged["Previous Schools_B"]]
print(df_different)
Name Age Birth_Country Previous Schools_A Previous Schools_B
1 nick 15 MEX 1 4.0
2 juli 14 CAN 0 NaN
and to find the rows where Previous Schools has not changed:
df_unchanged = df_merged[df_merged["Previous Schools_A"]==df_merged["Previous Schools_B"]]
print(df_unchanged)
Name Age Birth_Country Previous Schools_A Previous Schools_B
0 tom 10 USA 3 3.0
3 tom 19 NOR 1 1.0
If I were you, I'd stop here, because creating the final dataframe you want is going to have generic object column types because of the mix of strings and integers, which will limit its uses... but maybe you need it in that particular formattting for some reason. In which case, it's all about putting together these dataframe subsets in the right way to get your desired formatting. Here's one way.
First, initialize the final dataframe with the unchanged rows:
df_final = df_unchanged[["Name", "Age", "Birth_Country", "Previous Schools_A"]].copy()
df_final = df_final.rename(columns={"Previous Schools_A": "Previous Schools"})
print(df_final)
Name Age Birth_Country Previous Schools
0 tom 10 USA 3
3 tom 19 NOR 1
now process the entries that have changed between dataframes. There are two cases here: where the entries have changed (where Previous Schools_B is not NaN) and where the entrie is new (where Previous Schools_B is NaN). We'll deal with each in turn:
changed_entries = df_different[~pd.isnull(df_different["Previous Schools_B"])].copy()
changed_entries["Previous Schools"] = changed_entries["Previous Schools_A"].astype('str') + " --> " + changed_entries["Previous Schools_B"].astype('int').astype('str')
changed_entries = changed_entries.drop(columns=["Previous Schools_A", "Previous Schools_B"])
print(changed_entries)
Name Age Birth_Country Previous Schools
1 nick 15 MEX 1 --> 4
and now process the entries that are completely new:
new_entries = df_different[pd.isnull(df_different["Previous Schools_B"])].copy()
new_entries = "NaN --> " + new_entries[["Name", "Age", "Birth_Country", "Previous Schools_A"]].astype('str')
new_entries = new_entries.rename(columns={"Previous Schools_A": "Previous Schools"})
print(new_entries)
Name Age Birth_Country Previous Schools
2 NaN --> juli NaN --> 14 NaN --> CAN NaN --> 0
and finally, concatenate all the dataframes:
df_final = pd.concat([df_final, changed_entries, new_entries])
print(df_final)
Name Age Birth_Country Previous Schools
0 tom 10 USA 3
3 tom 19 NOR 1
1 nick 15 MEX 1 --> 4
2 NaN --> juli NaN --> 14 NaN --> CAN NaN --> 0
Let's say that I have this dataframe with four columns : "Name", "Value", "Ccy" and "Group" :
import pandas as pd
Name = ['ID', 'Country', 'IBAN','Dan_Age', 'Dan_city', 'Dan_country', 'Dan_sex', 'Dan_Age', 'Dan_country','Dan_sex' , 'Dan_city','Dan_country' ]
Value = ['TAMARA_CO', 'GERMANY','FR56','18', 'Berlin', 'GER', 'M', '22', 'FRA', 'M', 'Madrid', 'ESP']
Ccy = ['','','','EUR','EUR','USD','USD','','CHF', '','DKN','']
Group = ['0','0','0','1','1','1','1','2','2','2','3','3']
df = pd.DataFrame({'Name':Name, 'Value' : Value, 'Ccy' : Ccy,'Group':Group})
print(df)
Name Value Ccy Group
0 ID TAMARA_CO 0
1 Country GERMANY 0
2 IBAN FR56 0
3 Dan_Age 18 EUR 1
4 Dan_city Berlin EUR 1
5 Dan_country GER USD 1
6 Dan_sex M USD 1
7 Dan_Age 22 2
8 Dan_country FRA CHF 2
9 Dan_sex M 2
10 Dan_city Madrid DKN 3
11 Dan_country ESP 3
I want to represent this data differently before saving it in a csv. I would like to group the duplicates in the column "Name" with the associates values in "Values" and "Ccy". I want that the data in the column "Value" and "Ccy" are stored in the row(index) defined by the column "Group". Like that I do not mixed the data.
Then if the name is in the "group" 0, it means that it is general data so I would like that the all the rows from this "Name" are filled with the same value.
So I would like to get this result :
ID_Value Country_Value IBAN_Value Dan_age Dan_age_Ccy Dan_city_Value Dan_city_Ccy Dan_sex_Value
1 TAMARA GER FR56 18 EUR Berlin EUR M
2 TAMARA GER FR56 22 M
3 TAMARA GER FR56 Madrid DKN
I can not find how to do the first part. With the code below, I do not get what I want evn if I remove the columns empty
g = df.groupby(['Name']).cumcount()
df = df.set_index([g,'Name']).unstack().sort_index(level=1, axis=1)
df.columns = df.columns.map(lambda x: f'{x[0]}_{x[1]}')
Anyone can help me !
Thank you
You can use the following. See comments in code for each step:
s = df.loc[df['Group'] == '0', 'Name'].tolist() # this variable will be used later according to Condition 2
df['Name'] = pd.Categorical(df['Name'], categories=df['Name'].unique(), ordered=True) #this preserves order before pivoting
df = df.pivot(index='Group', columns='Name') #transforms long-to-wide per expected output
for col in df.columns:
if col[1] in s: df[col] = df[col].shift().ffill() #Condition 2
df = df.iloc[1:].replace('',np.nan).dropna(axis=1, how='all').fillna('') #dataframe cleanup
df.columns = ['_'.join(col) for col in df.columns.swaplevel()] #column name cleanup
df
Out[1]:
ID_Value Country_Value IBAN_Value Dan_Age_Value Dan_city_Value \
Group
1 TAMARA_CO GERMANY FR56 18 Berlin
2 TAMARA_CO GERMANY FR56 22
3 TAMARA_CO GERMANY FR56 Madrid
Dan_country_Value Dan_sex_Value Dan_Age_Ccy Dan_city_Ccy \
Group
1 GER M EUR EUR
2 FRA M
3 ESP DKN
Dan_country_Ccy Dan_sex_Ccy
Group
1 USD USD
2 CHF
3
From there, you can drop columns you don't want, change strings from "TAMARA_CO" to "TAMARA", "GERMANY" to "GER", use reset_index(drop=True), etc.
You can do this quite easily with only 3 steps:
Split your data frame into 2 parts: the "general data" (which we want as a series) and the more specific data. Each data frame now contains the same kinds of information.
The key part of your problem: reorganizing the data. All you need is the pandas pivot function. It does exactly what you need!
Add the general information and the pivoted data back together.
# Split Data
general = df[df.Group == "0"].set_index("Name")["Value"].copy()
main_df = df[df.Group != "0"]
# Pivot Data
result = main_df.pivot(index="Group", columns=["Name"],
values=["Value", "Ccy"]).fillna("")
result.columns = [f"{c[1]}_{c[0]}" for c in result.columns]
# Create a data frame that has an identical row for each group
general_df = pd.DataFrame([general]*3, index=result.index)
general_df.columns = [c + "_Value" for c in general_df.columns]
# Merge the data back together
result = general_df.merge(result, on="Group")
The result given above does not give the exact column order you want, so you'd have to specify that manually with
final_cols = ["ID_Value", "Country_Value", "IBAN_Value",
"Dan_age_Value", "Dan_Age_Ccy", "Dan_city_Value",
"Dan_city_Ccy", "Dan_sex_Value"]
result = result[final_cols]