below is part of my data, currently, there's a requirement to change the old_data to the required one. Just use the below data for example.
df = pd.DataFrame({'old_data':['12-13A:A','12-13A:B','12-13A:C','12-13A:G','39-40:A','39-40:B','39-40:G','13A-19:A','13A-19:B',
'13A-19:C','13A-19:D','13A-19:E','13A-19:F','13A-19:G']})
The pre-defined rule is that the digit range difference of each group's old data is 2(like 39-40),3(like 12-13A), or 6(like 13A-19). And if the single digit of a number is 4, then we need to change it to the number before that number appending an 'A'. For example, the number 14, we need to change it to 13A, 23A means 24. If the old_data is 33-35:B, the required data shall be Bed 33A.
Appreciate you can give some ideas for how to modify the old_data column to the required_data column with Pandas, Thanks.
Essentially your data is range:alphabet_index.
Some helper functions. I will switch between your 'no-four-in-the-last-digit' integer system and the normal integer system
import re
def to_number_system(s):
return int(re.sub('3A$', '4', s))
def to_no_four_system(n):
return 'Bed ' + re.sub('4$', '3A', str(n))
The following function map your alphabetical indices to the Bed numbers generated by the range, or G to the range itself.
def do_the_job(df):
_range = df['_range'].iloc[0]
range_start, range_end = map(to_number_system, _range.split('-'))
numbers = map(to_no_four_system, range(range_start, range_end+1))
return df['index'].map(dict(zip('ABCDEF', numbers), G=_range))
df[['_range', 'index']] = df['old_data'].str.split(':', expand=True)
df['required_data'] = df.groupby('_range').apply(do_the_job).droplevel(0)
Take care of the formatting
df.drop(columns=['_range', 'index'])
Good morning,
I have been having troubles getting soap API data into google sheets. When i run the Soap request I get the data as shown in the image .
[output data][1]
Then i tried getting this data into a google sheets using different methods, unfortunately no solution so far has worked.
The solutions i have tried is pickling the data the setting it in a different file and pushing that file into google sheets.
The current solution I'm working on is setting the output_data in a pandas dataframe and pushing like that, this is the current code but this also doesn't seem to work. I will only leave out the credentials to authenticate with the API.
def pandas_to_sheets(pandas_df, sheet, clear = True):
# Updates all values in a workbook to match a pandas dataframe
if clear:
sheet.clear()
(row, col) = pandas_df.shape
cells = sheet.range("A1:{}".format(gspread.utils.rowcol_to_a1(row + 1, col)))
for cell, val in zip(cells, iter_pd(pandas_df)):
cell.value = val
sheet.update_cells(cells)
def iter_pd(df):
for val in list(df.columns):
yield val
for row in df.values:
for val in list(row):
if pd.isna(val):
yield ""
else:
yield val
optionsReportAffiliateSite = [ {'dateFrom' : '01-01-2020', }]
client = Client(wsdl)
client.service.authenticate(username, password, sandbox, locale, demo)
testReportAffiliateSite = client.service.getReportAffiliateSite(idCampaigns,optionsReportCampaign )
input_dict = zeep.helpers.serialize_object(testReportAffiliateSite)
df = pd.DataFrame(input_dict)
affliatesite = pd.DataFrame(df.values.tolist())[0]
reportdata = pd.DataFrame(df.values.tolist())[1]
pd.json_normalize(affliatesite)
pd.json_normalize(reportdata)
pd.concat([pd.json_normalize(affliatesite), pd.json_normalize(reportdata).reindex(pd.json_normalize(affliatesite).index)], axis=1)
wks = gc.open_by_key('1uPdi2w_1TajnKNN8G3uahgrSHLAPnbAHtHSPeaZN3y0').sheet1
pandas_to_sheets(pd.concat([pd.json_normalize(affliatesite), pd.json_normalize(reportdata).reindex(pd.json_normalize(affliatesite).index)], axis=1), wks)
This gives me the error "TypeError: Object of type Decimal is not JSON serializable"
Many thanks in advance.
[1]: https://i.stack.imgur.com/efOEN.png
The decimal values in the response you're getting cannot be serialized to JSON.
Because of this, you should transform this decimal values to another type which can be serialized. For example, float. So you can do the following:
Define this function, to check if an element is a decimal and return it converted to float:
def f(v):
if isinstance(v, Decimal):
return float(v)
else:
return v
Iterate through your list and call the previous function for every value in it, using this:
soapResponse = map(lambda el : {k: f(v) for k, v in el.items()}, soapResponse)
Note:
Import decimal via from decimal import Decimal.
Reference:
decimal
float
map
Python JSON serialize a Decimal object
I am trying to only show the dates to the uses of this Python application. For some reason, the code returns responses like "Date" and "None" from the spreadsheet. Date is in the column that I am trying to draw the dates from. Here is the code:
sh = gc.open("Deposits")
worksheet = sh.worksheet("Sheet2")
values_list = worksheet.col_values(3)
set = set(values_list)
result = list(set)
print "Here are all the possible dates to check:",result
Result:
['3/10/2012', '2/18/2013', '3/18/2011', '3/17/2010', 'Date', None, '2/9/2010']
How do I get this function to only return the dates and exclude 'Date' and 'None'?
Just subtract a set that contains the things you don't want to include.
myset = set(values_list) - {None, 'Date'}
Also, don't use variable names that are already assigned to built-in functions, like set, or you'll run into problems when you want to use that built-in function.
You can use a list comprehension to get rid of "Date" and None
a = ['3/10/2012', '2/18/2013', '3/18/2011', '3/17/2010', 'Date', None, '2/9/2010']
r = list(set([i for i in a if i not in("Date",None)]))
['3/10/2012', '2/18/2013', '3/18/2011', '3/17/2010', '2/9/2010']
I would like to simulate the Excel autofit function in Python's xlsxwriter. According to this url, it is not directly supported:
http://xlsxwriter.readthedocs.io/worksheet.html
However, it should be quite straightforward to loop through each cell on the sheet and determine the maximum size for the column and just use worksheet.set_column(row, col, width) to set the width.
The complications that is keeping me from just writing this are:
That URL does not specify what the units are for the third argument to set_column.
I can not find a way to measure the width of the item that I want to insert into the cell.
xlsxwriter does not appear to have a method to read back a particular cell. This means I need to keep track of each cell width as I write the cell. It would be better if I could just loop through all the cells, that way a generic routine could be written.
[NOTE: as of Jan 2023 xslxwriter added a new method called autofit. See jmcnamara's answer below]
As a general rule, you want the width of the columns a bit larger than the size of the longest string in the column. The with of 1 unit of the xlsxwriter columns is about equal to the width of one character. So, you can simulate autofit by setting each column to the max number of characters in that column.
Per example, I tend to use the code below when working with pandas dataframes and xlsxwriter.
It first finds the maximum width of the index, which is always the left column for a pandas to excel rendered dataframe. Then, it returns the maximum of all values and the column name for each of the remaining columns moving left to right.
It shouldn't be too difficult to adapt this code for whatever data you are using.
def get_col_widths(dataframe):
# First we find the maximum length of the index column
idx_max = max([len(str(s)) for s in dataframe.index.values] + [len(str(dataframe.index.name))])
# Then, we concatenate this to the max of the lengths of column name and its values for each column, left to right
return [idx_max] + [max([len(str(s)) for s in dataframe[col].values] + [len(col)]) for col in dataframe.columns]
for i, width in enumerate(get_col_widths(dataframe)):
worksheet.set_column(i, i, width)
I agree with Cole Diamond. I needed to do something very similar, it worked fine for me. where self.columns is my list of columns
def set_column_width(self):
length_list = [len(x) for x in self.columns]
for i, width in enumerate(length_list):
self.worksheet.set_column(i, i, width)
That URL does not specify what the units are for the third argument to set_column.
The column widths are given in multiples of the width of the '0' character in the font Calibri, size 11 (that's the Excel standard).
I can not find a way to measure the width of the item that I want to insert into the cell.
In order to get a handle on the exact width of a string, you can use tkinter's ability to measure string lengths in pixels, depending on the font/size/weight/etc. If you define a font, e.g.
reference_font = tkinter.font.Font(family='Calibri', size=11)
you can afterwards use its measure method to determine string widths in pixels, e.g.
reference_font.measure('This is a string.')
In order to do this for a cell from your Excel table, you need to take its format into account (it contains all the information on the used font). That means, if you wrote something to your table using worksheet.write(row, col, cell_string, format), you can get the used font like this:
used_font = tkinter.font.Font(family = format.font_name,
size = format.font_size,
weight = ('bold' if format.bold else 'normal'),
slant = ('italic' if format.italic else 'roman'),
underline = format.underline,
overstrike = format.font_strikeout)
and afterwards determine the cell width as
cell_width = used_font.measure(cell_string+' ')/reference_font.measure('0')
The whitespace is added to the string to provide some margin. This way the results are actually very close to Excel's autofit results, so that I assume Excel is doing just that.
For the tkinter magic to work, a tkinter.Tk() instance (a window) has to be open, therefore the full code for a function that returns the required width of a cell would look like this:
import tkinter
import tkinter.font
def get_cell_width(cell_string, format = None):
root = tkinter.Tk()
reference_font = tkinter.font.Font(family='Calibri', size=11)
if format:
used_font = tkinter.font.Font(family = format.font_name,
size = format.font_size,
weight = ('bold' if format.bold else 'normal'),
slant = ('italic' if format.italic else 'roman'),
underline = format.underline,
overstrike = format.font_strikeout)
else:
used_font = reference_font
cell_width = used_font.measure(cell_string+' ')/reference_font.measure('0')
root.update_idletasks()
root.destroy()
return cell_width
Of course you would like to get the root handling and reference font creation out of the function, if it is meant to be executed frequently. Also, it might be faster to use a lookup table format->font for your workbook, so that you do not have to define the used font every single time.
Finally, one could take care of line breaks within the cell string:
pixelwidths = (used_font.measure(part) for part in cell_string.split('\n'))
cell_width = (max(pixelwidths) + used_font.measure(' '))/reference_font.measure('0')
Also, if you are using the Excel filter function, the dropdown arrow symbol needs another 18 pixels (at 100% zoom in Excel). And there might be merged cells spanning multiple columns... A lot of room for improvements!
xlsxwriter does not appear to have a method to read back a particular cell. This means I need to keep track of each cell width as I write the cell. It would be better if I could just loop through all the cells, that way a generic routine could be written.
If you do not like to keep track within your own data structure, there are at least three ways to go:
(A) Register a write handler to do the job:
You can register a write handler for all standard types. In the handler function, you simply pass on the write command, but also do the bookkeeping wrt. column widths. This way, you only need to read and set the optimal column width in the end (before closing the workbook).
# add worksheet attribute to store column widths
worksheet.colWidths = [0]*number_of_used_columns
# register write handler
for stdtype in [str, int, float, bool, datetime, timedelta]:
worksheet.add_write_handler(stdtype, colWidthTracker)
def colWidthTracker(sheet, row, col, value, format):
# update column width
sheet.colWidths[col] = max(sheet.colWidths[col], get_cell_width(value, format))
# forward write command
if isinstance(value, str):
if value == '':
sheet.write_blank(row, col, value, format)
else:
sheet.write_string(row, col, value, format)
elif isinstance(value, int) or isinstance(value, float):
sheet.write_number(row, col, value, format)
elif isinstance(value, bool):
sheet.write_boolean(row, col, value, format)
elif isinstance(value, datetime) or isinstance(value, timedelta):
sheet.write_datetime(row, col, value, format)
else:
raise TypeError('colWidthTracker cannot handle this type.')
# and in the end...
for col in columns_to_be_autofitted:
worksheet.set_column(col, col, worksheet.colWidths[col])
(B) Use karolyi's answer above to go through the data stored within XlsxWriter's internal variables. However, this is discouraged by the module's author, since it might break in future releases.
(C) Follow the recommendation of jmcnamara: Inherit from and override the default worksheet class and add in some autofit code, like this example: xlsxwriter.readthedocs.io/example_inheritance2.html
I recently ran into this same issue and this is what I came up with:
r = 0
c = 0
for x in list:
worksheet.set_column('{0}:{0}'.format(chr(c + ord('A'))), len(str(x)) + 2)
worksheet.write(r, c, x)
c += 1
In my example r would be the row number you are outputting to, c would be the column number you are outputting to (both 0 indexed), and x would be the value from list that you are wanting to be in the cell.
the '{0}:{0}'.format(chr(c + ord('A'))) piece takes the column number provided and converts it to the column letter accepted by xlsxwriter, so if c = 0 set_column would see 'A:A', if c = 1 then it would see 'B:B', and so on.
the len(str(x)) + 2 piece determines the length of the string you are trying to output then adds 2 to it to ensure that the excel cell is wide enough as the length of the string does not exactly correlate to the width of the cell. You may want to play with rather you add 2 or possibly more depending on your data.
The units that xlsxwriter accepts is a little harder to explain. When you are in excel and you hover over where you can change the column width you will see Width: 8.43 (64 pixels). In this example the unit it accepts is the 8.43, which I think is centimeters? But excel does not even provide a unit, at least not explicitly.
Note: I have only tried this answer on excel files that contain 1 row of data. If you will have multiple rows, you will need to have a way to determine which row will have the 'longest' information and only apply this to that row. But if each column will be roughly the same size regardless of row, then this should work fine for you.
Good luck and I hope this helps!
Update from January 2023.
XlsxWriter 3.0.6+ now supports a autofit() worksheet method:
from xlsxwriter.workbook import Workbook
workbook = Workbook('autofit.xlsx')
worksheet = workbook.add_worksheet()
# Write some worksheet data to demonstrate autofitting.
worksheet.write(0, 0, "Foo")
worksheet.write(1, 0, "Food")
worksheet.write(2, 0, "Foody")
worksheet.write(3, 0, "Froody")
worksheet.write(0, 1, 12345)
worksheet.write(1, 1, 12345678)
worksheet.write(2, 1, 12345)
worksheet.write(0, 2, "Some longer text")
worksheet.write(0, 3, "http://ww.google.com")
worksheet.write(1, 3, "https://github.com")
# Autofit the worksheet.
worksheet.autofit()
workbook.close()
Output:
Or using Pandas:
import pandas as pd
# Create a Pandas dataframe from some data.
df = pd.DataFrame({
'Country': ['China', 'India', 'United States', 'Indonesia'],
'Population': [1404338840, 1366938189, 330267887, 269603400],
'Rank': [1, 2, 3, 4]})
# Order the columns if necessary.
df = df[['Rank', 'Country', 'Population']]
# Create a Pandas Excel writer using XlsxWriter as the engine.
writer = pd.ExcelWriter('pandas_autofit.xlsx', engine='xlsxwriter')
df.to_excel(writer, sheet_name='Sheet1', index=False)
# Get the xlsxwriter workbook and worksheet objects.
workbook = writer.book
worksheet = writer.sheets['Sheet1']
worksheet.autofit()
# Close the Pandas Excel writer and output the Excel file.
writer.close()
Output:
Cole Diamond's answer is awesome. I just updated the subroutine to handle multiindex rows and columns.
def get_col_widths(dataframe):
# First we find the maximum length of the index columns
idx_max = [max([len(str(s)) for s in dataframe.index.get_level_values(idx)] + [len(str(idx))]) for idx in dataframe.index.names]
# Then, we concatenate this to the max of the lengths of column name and its values for each column, left to right
return idx_max + [max([len(str(s)) for s in dataframe[col].values] + \
[len(str(x)) for x in col] if dataframe.columns.nlevels > 1 else [len(str(col))]) for col in dataframe.columns]
There is another workaround to simulate Autofit that I've found on the Github site of xlsxwriter. I've modified it to return the approximate size of horizontal text (column width) or 90° rotated text (row height):
from PIL import ImageFont
def get_cell_size(value, font_name, font_size, dimension="width"):
""" value: cell content
font_name: The name of the font in the target cell
font_size: The size of the font in the target cell """
font = ImageFont.truetype(font_name, size=font_size)
(size, h) = font.getsize(str(value))
if dimension == "height":
return size * 0.92 # fit value experimentally determined
return size * 0.13 # fit value experimentally determined
This doesn't address bold text or other format elements that might affect the text size. Otherwise it works pretty well.
To find the width for your columns for autofit:
def get_col_width(data, font_name, font_size, min_width=1):
""" Assume 'data' to be an iterable (rows) of iterables (columns / cells)
Also, every cell is assumed to have the same font and font size.
Returns a list with the autofit-width per column """
colwidth = [min_width for col in data[0]]
for x, row in enumerate(data):
for y, value in enumerate(row):
colwidth[y] = max(colwidth[y], get_cell_size(value, font_name, font_size))
return colwidth
My version that will go over the one worksheet and autoset the field lengths:
from typing import Optional
from xlsxwriter.worksheet import (
Worksheet, cell_number_tuple, cell_string_tuple)
def get_column_width(worksheet: Worksheet, column: int) -> Optional[int]:
"""Get the max column width in a `Worksheet` column."""
strings = getattr(worksheet, '_ts_all_strings', None)
if strings is None:
strings = worksheet._ts_all_strings = sorted(
worksheet.str_table.string_table,
key=worksheet.str_table.string_table.__getitem__)
lengths = set()
for row_id, colums_dict in worksheet.table.items(): # type: int, dict
data = colums_dict.get(column)
if not data:
continue
if type(data) is cell_string_tuple:
iter_length = len(strings[data.string])
if not iter_length:
continue
lengths.add(iter_length)
continue
if type(data) is cell_number_tuple:
iter_length = len(str(data.number))
if not iter_length:
continue
lengths.add(iter_length)
if not lengths:
return None
return max(lengths)
def set_column_autowidth(worksheet: Worksheet, column: int):
"""
Set the width automatically on a column in the `Worksheet`.
!!! Make sure you run this function AFTER having all cells filled in
the worksheet!
"""
maxwidth = get_column_width(worksheet=worksheet, column=column)
if maxwidth is None:
return
worksheet.set_column(first_col=column, last_col=column, width=maxwidth)
just call set_column_autowidth with the column.
Some of the solutions given here were too elaborate for the rather simple thing that I was looking for: every column had to be sized so that all its values fits nicely. So I wrote my own solution. It basically iterates over all columns, and for each column it gets all string values (including the column name itself) and then takes the longest string as the maximal width for that column.
# Set the width of the columns to the max. string length in that column
# ~ simulates Excel's "autofit" functionality
for col_idx, colname in enumerate(df.columns):
max_width = max([len(colname)]+[len(str(s)) for s in df[colname]])
worksheet.set_column(col_idx, col_idx, max_width+1) # + 1 to add some padding
Here is a version of code that supports MultiIndex for row and column - it is not pretty but works for me. It expands on #cole-diamond answer:
def _xls_make_columns_wide_enough(dataframe, worksheet, padding=1.1, index=True):
def get_col_widths(dataframe, padding, index):
max_width_idx = []
if index and isinstance(dataframe.index, pd.MultiIndex):
# Index name lengths
max_width_idx = [len(v) for v in dataframe.index.names]
# Index value lengths
for column, content in enumerate(dataframe.index.levels):
max_width_idx[column] = max(max_width_idx[column],
max([len(str(v)) for v in content.values]))
elif index:
max_width_idx = [
max([len(str(s))
for s in dataframe.index.values] + [len(str(dataframe.index.name))])
]
if isinstance(dataframe.columns, pd.MultiIndex):
# Take care of columns - headers first.
max_width_column = [0] * len(dataframe.columns.get_level_values(0))
for level in range(len(dataframe.columns.levels)):
values = dataframe.columns.get_level_values(level).values
max_width_column = [
max(v1, len(str(v2))) for v1, v2 in zip(max_width_column, values)
]
# Now content.
for idx, col in enumerate(dataframe.columns):
max_width_column[idx] = max(max_width_column[idx],
max([len(str(v)) for v in dataframe[col].values]))
else:
max_width_column = [
max([len(str(s)) for s in dataframe[col].values] + [len(col)])
for col in dataframe.columns
]
return [round(v * padding) for v in max_width_idx + max_width_column]
for i, width in enumerate(get_col_widths(dataframe, padding, index)):
worksheet.set_column(i, i, width)
Openpyxl easily handles this task. Just install the module and insert the below line of code in your file
# Imorting the necessary modules
try:
from openpyxl.cell import get_column_letter
except ImportError:
from openpyxl.utils import get_column_letter
from openpyxl.utils import column_index_from_string
from openpyxl import load_workbook
import openpyxl
from openpyxl import Workbook
for column_cells in sheet.columns:
new_column_length = max(len(str(cell.value)) for cell in column_cells)
new_column_letter = (get_column_letter(column_cells[0].column))
if new_column_length > 0:
sheet.column_dimensions[new_column_letter].width = new_column_length*1.23