Develop Reference

Pandas to Excel - How to apply conditional formatting based on two columns

I have a multi sheet excel file like the one pictured below. I want to highlight with the condition:
if value 'app' in column 'Kind' matches with value 'v6.0' in column 'NetFrameworkVersion'
then highlight it yellow
if value 'functionapp' in column 'Kind' matches with value 'v4.0' in column 'NetFrameworkVersion'
then highlight it green
else highlight it blue
import pandas as pd
import pathlib
import xlsxwriter
with pd.ExcelWriter('*/environment.xlsx' , engine='xlsxwriter') as writer:
for filename in pathlib.Path('*/FD').glob('*.csv'):
df = pd.read_csv(filename)
df_custom = df.filter(['Kind', 'NetFrameworkVersion', 'Use32BitWorkerProcess', 'AlwaysOn' ]) #Select column and arrange custom
sheetname = filename.stem.split('-')[3] #Set sheet name save as short name
df_custom.style.to_excel(writer, sheet_name=sheetname,index=True, header = True) #write out file excel after read all data from csv files
#set border#
workbook = writer.book
worksheet = writer.sheets[sheetname]
border_fmt = workbook.add_format({'bottom':1, 'top':1, 'left':1, 'right':1})
worksheet.conditional_format(xlsxwriter.utility.xl_range(0, 0, len(df), len(df_custom.columns)), {'type': 'no_errors', 'format': border_fmt})
worksheet.set_column('C1:Z200', 25) #set range column width
worksheet.set_column('B:B', 35) #set 1 column width
red_format = workbook.add_format({'bg_color': '#FFC7CE',
'font_color': '#9C0006'})
worksheet.conditional_format('F1:F1000', {'type': 'text',
'criteria': 'containing',
'value': 'True',
'format': red_format})
worksheet.conditional_format('G1:G100', {'type': 'text',
'criteria': 'containing',
'value': 'False',
'format': red_format})
writer.save()
Example Picture:

Let df be the DataFrame from your picture.
Something like this should work:
import pandas as pd # Jinja2 must be installed
def styler(series):
color_attr = ''
if series['Kind'] == 'app' and series['NetFrameworkVersion'] == 'v6.0':
color_attr = 'background-color: yello'
elif series['Kind'] == 'functionapp' and series['NetFrameworkVersion'] == 'v4.0':
color_attr = 'background-color: green'
else:
color_attr = 'background-color: blue'
return pd.Series(color_attr, series.index)
left_df_p = df[['Kind', 'NetFrameworkVersion']]
left_df_p.style.apply(styler, axis=1) # iterates over every row
colored_df = left_df_p.join(df[['Use32BitWorkerProcess', 'AlwaysOn']])
Next time, please provide a Minimal, Reproducible Example, so that people can actually test the code. Instead of posting a picture, it is better to call and post the output of df.to_dict.

Output to screen and csv format python

I have a python nested dictionary output, I have been able to remove the first set of cruly brackets using RocketDict, but 1) I can't remove the second set of curly brackets 2)I tried to export it to a csv file giving the column names and that doesn't work because I can't figure out how to get the int#/# values that increment in the rows. For Example here was my initial output:
Before RocketDict:
{ intx/x : {'value1: 'A', 'value2: 'B', value3: 'C'},
inty/y : {'value1: 'X', 'value2: 'Y', value3: 'Z'}}
After the RocketDict:
intx/x : {'value1: 'A', 'value2: 'B', value3: 'C'},
inty/y : {'value1: 'X', 'value2: 'Y', value3: 'Z'}
Desired output:
intx/x : 'value1: 'A', 'value2: 'B', value3: 'C',
inty/y : 'value1: 'X', 'value2: 'Y', value3: 'Z'
Desired output to the csv:
Here is the full script:
results = requests.get(url, headers=headers)
inventory = results.json()
data = inventory['config']
class RocketDict(UserDict):
def __str__(self):
r = ['']
r.extend(['\t{} : {}'.format(k, v)
for k, v in self.items()])
return ',\n'.join(r)
if __name__ == '__main__':
#standard dict object
# inventory = {('key-%02d' % v): v for v in range(1, 10)}
# print(inventory, '\n')
# Wrap that dict object into a RocketDict.
d2 = RocketDict(data)
print(d2)
csv_columns = ['value1','value2','value3']
dict_data = d2
csv_file = 'mycsv.csv'
try:
with open(csv_file, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=csv_columns)
writer.writeheader()
for data in dict_data:
writer.writerow(d2)
except IOError:
print("I/O error")

Use pandas -
RocketDict ={ 'intx/x' : {'value1': 'A', 'value2': 'B', 'value3': 'C'},
'inty/y' : {'value1': 'X', 'value2': 'Y', 'value3': 'Z'}}
import pandas as pd
pd.DataFrame(RocketDict).transpose().to_csv('out.csv', index =True)

drop a DataFrame column in python

I desperately need help here. I am trying to get the dimension of a dataframe. I always get 31 columns instead of 30: Value should be 30, found 31. I tried to reset_index(drop = True) but I still get the same error. any help is appreciated. Stay safe.
def read_data(dataset_id):
data = None
# Begin CODE
if dataset_id == 'breast_cancer':
disease = 'breast_cancer'
datafile = 'wdbc.data'
bc_columns = ['ptid', 'diagnosis', 'mean_radius', 'mean_texture',
'mean_perimeter', 'mean_area',
'mean_smoothness', 'mean_compactness', 'mean_concavity',
'mean_concave_pts', 'mean_symmetry ',
'mean_fractal_dim', 'std_err_radius', 'std_err_texture',
'std_err_perimeter', 'std_err_area',
'std_err_smoothness', 'std_err_compactness',
'std_err_concavity', 'std_err_concave_pts',
'std_err_symmetry ', 'std_err_fractal_dim', 'worst_radius',
'worst_texture', 'worst_perimeter',
'worst_area', 'worst_smoothness', 'worst_compactness',
'worst_concavity', 'worst_concave_pts',
'worst_symmetry ', 'worst_fractal_dim']
data = pd.read_csv(datafile, skipinitialspace=True, names=bc_columns)
data.drop(labels=['ptid'], axis=1, inplace=True)
bc_diag_class = get_class_list_dict(data['diagnosis'])
elif dataset_id == 'hyperthyroidism':
disease = 'hyperthyroidism'
datafile1 = 'allhyper.data' # tab delimited, no header
datafile2 = 'allhyper.test' # comma delimited, no header
ht_columns = ['age', 'Gender', 'on thyroxine', 'query on thyroxine', 'on
antithyroid medication', 'sick',
'pregnant', 'thyroid surgery', 'I131 treatment', 'query
hypothyroid', 'query hyperthyroid',
'lithium', 'goitre', 'tumor', 'hypopituitary', 'psych',
'TSH measured', 'TSH', 'T3 measured',
'T3', 'TT4 measured', 'TT4', 'T4U measured', 'T4U', 'FTI
measured', 'FTI', 'TBG measured', 'TBG',
'referral source', 'diag_class']
data1 = pd.read_csv(datafile1, sep='\t', skipinitialspace=True,
names=ht_columns)
data2 = pd.read_csv(datafile2, skipinitialspace=True, names=ht_columns)
data = data1.append(data2, ignore_index=True)
data = data.replace(to_replace='?', value=float('nan'))
data[['diag_class', 'ptid']] = data['diag_class'].str.split(pat='.\|',
expand=True)
diag_class = data['diag_class']
data.drop(labels=['diag_class', 'ptid'], axis=1, inplace=True)
data.insert(0, 'diag_class', diag_class)
data[['age', 'TSH', 'T3', 'TT4', 'T4U', 'FTI', 'TBG']] \
= data[['age', 'TSH', 'T3', 'TT4', 'T4U', 'FTI',
'TBG']].apply(pd.to_numeric)
elif dataset_id == 'cervical_cancer':
disease = 'cervical_cancer'
datafile = 'risk_factors_cervical_cancer.csv'
cc_columns = ('Age', 'Num_sex_partners', 'First_sex_intercourse',
'Num_pregnancies',
'Smokes', 'Smokes_years', 'Smokes_packs_year',
'Hormonal_Contraceps',
'Hormonal_Contraceps_years', 'IUD', 'IUD_years', 'STD',
'STD_number',
'STD_condylomatosis', 'STDscervical_condylomatosis',
'STD_vaginal_condylomatosis',
'STD_vulvo_perin_condylomatosis', 'STD_syphilis',
'STD_pelvic_inflam_disease',
'STD_genital_herpes', 'STD_molluscum_contagiosum',
'STD_AIDS', 'STD_HIV', 'STD_HepB',
'STD_HPV', 'STD_Num_diagnosis',
'STD_Time_since_first_diag', 'STDs_Time_since_last_diag',
'Dx_Cancer', 'Dx_CIN', 'Dx_HPV', 'Dx', 'Hinselmann', 'Schiller',
'Citology', 'Biopsy')
data = pd.read_csv(datafile, skipinitialspace=True)
data.columns = cc_columns
data = data.replace(to_replace='?', value=float('nan'))
biopsy_class = data['Biopsy']
data.drop(labels=['Dx_Cancer', 'Dx_CIN', 'Dx_HPV', 'Dx', 'Hinselmann',
'Schiller', 'Citology', 'Biopsy'],
axis=1, inplace=True)
data.insert(0, 'Biopsy', biopsy_class)
data[['Num_sex_partners', 'First_sex_intercourse', 'Num_pregnancies',
'Smokes_years', 'Smokes_packs_year',
'Hormonal_Contraceps_years', 'IUD_years',
'STD_number', 'STD_Time_since_first_diag',
'STDs_Time_since_last_diag']] \
= data[['Num_sex_partners', 'First_sex_intercourse',
'Num_pregnancies', 'Smokes_years', 'Smokes_packs_year',
'Hormonal_Contraceps_years', 'IUD_years',
'STD_number', 'STD_Time_since_first_diag',
'STDs_Time_since_last_diag']].apply(pd.to_numeric)
elif dataset_id == 'liver_cancer':
disease = 'liver_cancer'
datafile = 'Indian Liver Patient Dataset (ILPD).csv' # comma delimited,
no header
ld_columns = ['Age', 'Gender', 'TB', 'DB', 'Alkphos', 'Sgpt', 'Sgot',
'TP', 'ALB', 'A/G Ratio', 'Selector']
data = pd.read_csv(datafile, skipinitialspace=True, names=ld_columns)
data.loc[data['Gender'] == 'Male', 'Gender'] = 'M'
data.loc[data['Gender'] == 'Female', 'Gender'] = 'F'
selector_class = data['Selector']
data.drop(labels=['Selector'], axis=1, inplace=True)
data.insert(0, 'Selector', selector_class)
data.reset_index(drop=True, inplace=True)
# End CODE
print(data.head(20))
return data
def dimensions(dataset_id, dataset):
dim = None
# dim = dataset.shape
num_inst = len(dataset)
num_feat = len(dataset.iloc[0].reset_index())
dim = (num_inst, num_feat)
return dim

If you want to drop a column from DataFrame, You can do like this.
If you want to drop single column:
df.drop(['column_name'], axis = 1)
If you want to drop multiple columns:
df.drop(['Column1', 'Column2'], axis = 1)
If you want to drop based on some other condition instead of column name. You can comment below. I'll update the answer accordingly. Hope it helps!.

how to draw a beautiful colorful table with pandas or other package in Python?

I am trying to draw a colorful table, like this.
this post provides an approach.
from datetime import datetime, timedelta
import pandas as pd
name = ['Diego', 'Luis', 'Vidal', 'John', 'Yusef']
id = ['b000000005', 'b000000015', 'b000000002', 'b000000011', 'b000000013']
cel = [7878, 6464, 1100, 4545, 1717]
date = pd.to_datetime(['2017-05-31 20:53:00', '2017-05-11 20:53:00', '2017-05-08 20:53:00',
'2017-06-06 20:53:00', '2017-06-06 20:53:00'])
df = pd.DataFrame({'Name':name,'ID':id,'Cel':cel,'Date':date})
def color(val):
if val < datetime.now():
color = 'green'
elif val > datetime.now():
color = 'yellow'
elif val > (datetime.now() + timedelta(days=60)):
color = 'red'
return 'background-color: %s' % color
df.style.applymap(color, subset=['Date'])
exactly same code from that post, produces a different output.
the borders are missing, the color is also different from the one in the post.
what am i missing?

First use pandas style for set background colors with custom function and then Styler.set_table_styles for set css styles:
df = pd.DataFrame({'Red':[1,1,0,0,0],'Yellow':[0,0,1,0,1],'Green':[0,0,0,1,0]})
print (df)
def color(x):
c1 = 'background-color: green'
c2 = 'background-color: yellow'
c3 = 'background-color: red'
c4 = ''
m = x == 1
print (m)
df1 = pd.DataFrame(c4, index=x.index, columns=x.columns)
df1.loc[m['Red'], 'Red'] = c1
df1.loc[m['Yellow'], 'Yellow'] = c2
df1.loc[m['Green'], 'Green'] = c3
return df1
df.style.apply(color,axis=None).set_table_styles(
[{
'selector': 'th',
'props': [
('background-color', 'blue'),
('color', 'white'),
('border-color', 'black'),
('border-style ', 'solid'),
('border-width','1px')]
},
{
'selector': 'td',
'props': [
('border-color', 'black'),
('border-style ', 'solid'),
('border-width','1px')]
},
{'selector': '.row_heading',
'props': [('display', 'none')]},
{'selector': '.blank.level0',
'props': [('display', 'none')]}])

datetime.now() is different from the linked post (May2017) and your post (May2019), hence the difference in color.
You don't see the lines because your jupyter theme default's different. The code only colors the target cells. Here's the output of the same code on my computer:

reportlab dynamic data-driven header outputs wrong subtitle

I have created some fictitious, though representative, clinical trial type data using Pandas, and now come to some test reporting in ReportLab.
The data has a block (~50 rows) where the treatment column is 'Placebo' and the same amount where the treatment is 'Active'. I simply want to list the data using a sub-heading of 'Treatment Group: Placebo' for the first set and 'Treatment Group: Active' for the second.
There are some hits on a similar topic, and, indeed I've used one of the suggested techniques, namely to extend the arguments of a header functions using partial from functools.
title1 = "ACME Corp CONFIDENTIAL"
title2 = "XYZ123 / Anti-Hypertensive Draft"
title3 = "Protocol XYZ123"
title4 = "Study XYZ123"
title5 = "Listing of Demographic Data by Treatment Arm"
title6 = "All subjects"
def title(canvas, doc, bytext):
canvas.saveState()
canvas.setFont(styleN.fontName, styleN.fontSize)
canvas.drawString(DOCMARGIN, PAGE_HEIGHT*.975, title1)
canvas.drawString(DOCMARGIN, PAGE_HEIGHT*.950, title2)
canvas.drawString(DOCMARGIN, PAGE_HEIGHT*.925, title3)
canvas.drawCentredString(PAGE_WIDTH/2.0, PAGE_HEIGHT*.900, title4)
canvas.drawCentredString(PAGE_WIDTH/2.0, PAGE_HEIGHT*.875, title5)
canvas.drawCentredString(PAGE_WIDTH/2.0, PAGE_HEIGHT*.850, title6)
canvas.drawString(DOCMARGIN, PAGE_HEIGHT*.825, "Treatment Group:" + bytext)
canvas.restoreState()
This is then called as follows. n_groups has the value of 2 from a summary query and 0 maps to 'Placebo' and 1 maps to active.
def build_pdf(doc):
ptemplates = []
for armcd in range(n_groups):
ptemplates.append(PageTemplate(id = 'PT' + str(armcd), frames = [dataFrame,],
onPage = partial(title, bytext=t_dict[armcd]),
onPageEnd = foot))
doc.addPageTemplates(ptemplates)
elements = []
for armcd in range(n_groups):
elements.append(NextPageTemplate('PT' + str(armcd)))
sublist = [t for t in lista if t[0] == (armcd+1)]
sublist.insert(0,colheads)
data_table = Table(sublist, 6*[40*mm], len(sublist)*[DATA_CELL_HEIGHT], repeatRows=1)
data_table.setStyle(styleC)
elements.append(data_table)
elements.append(PageBreak())
doc.build(elements)
The report produces 6 pages. The first 3 pages of placebo data are correct, pages 5 & 6 of active data are correct, but page 4 - which should be the first page of the second 'active' group has the sub-title 'Treatment Group: Placebo'.
I have re-organized the order of the statements multiple times, but can't get Page 4 to sub-title correctly. Any help, suggestions or magic would be much appreciated.
[Edit 1: sample data structure]
The 'top' of the data starts as:
[
[1, 'Placebo', '000001-000015', '1976-09-20', 33, 'F', 'Black'],
[1, 'Placebo', '000001-000030', '1959-04-26', 50, 'M', 'Asian'],
[1, 'Placebo', '000001-000031', '1946-02-07', 64, 'F', 'Asian'],
[1, 'Placebo', '000001-000046', '1947-11-08', 62, 'M', 'Asian'],
etc for 50 rows, then continues with
[2, 'Active', '000001-000002', '1962-02-28', 48, 'F', 'Black'],
[2, 'Active', '000001-000008', '1975-10-20', 34, 'M', 'Black'],
[2, 'Active', '000001-000013', '1959-01-19', 51, 'M', 'White'],
[2, 'Active', '000001-000022', '1962-01-12', 48, 'F', 'Black'],
[2, 'Active', '000001-000036', '1976-10-17', 33, 'F', 'Asian'],
[2, 'Active', '000001-000045', '1980-12-31', 29, 'F', 'White'],
for another 50.
The column header inserted is:
['Treatment Arm Code',
'Treatment Arm',
'Site ID - Subject ID',
'Date of Birth',
'Age (Years)',
'Gender',
'Ethnicity'],
[Edit 2: A solution - move the PageBreak() and make it conditional:]
def build_pdf(doc):
ptemplates = []
for armcd in range(n_groups):
ptemplates.append(PageTemplate(id = 'PT' + str(armcd), frames = [dataFrame,],
onPage = partial(title, bytext=t_dict[armcd]),
onPageEnd = foot))
doc.addPageTemplates(ptemplates)
elements = []
for armcd in range(n_groups):
elements.append(NextPageTemplate('PT' + str(armcd)))
if armcd > 0:
elements.append(PageBreak())
sublist = [t for t in lista if t[0] == (armcd+1)]
sublist.insert(0,colheads)
data_table = Table(sublist, 6*[40*mm], len(sublist)*[DATA_CELL_HEIGHT], repeatRows=1)
data_table.setStyle(styleC)
elements.append(data_table)
doc.build(elements)