I have a list of dictionaries:
data = [{'average': 2, 'day': '2022-01-01'},
{'average': 3, 'day': '2022-01-02'},
{'average': 5, 'day': '2022-01-03'},
{'sum': 8, 'day': '2022-01-01'},
{'sum': 15, 'day': '2022-01-02'},
{'sum': 9, 'day': '2022-01-03'},
{'total_value': 19, 'day': '2022-01-01'},
{'total_value': 99, 'day': '2022-01-02'},
{'total_value': 15, 'day': '2022-01-03'}]
I want my output as:
output = [{'average': 2, 'sum': 8, 'total_value': 19, 'day': '2022-01-01'},
{'average': 3, 'sum': 15, 'total_value': 99, 'day': '2022-01-02'},
{'average': 5, 'sum': 9, 'total_value': 15, 'day': '2022-01-03'}]
The output puts the values together based off their date. My approaches so far have been to try and separate everything out into different dictionaries (date_dict, sum_dict, etc.) and then bringing them all together, but that doesn't seem to work and is extremely sloppy.
You could iterate over data and create a dictionary using day as key:
data = [{'average': 2, 'day': '2022-01-01'},
{'average': 3, 'day': '2022-01-02'},
{'average': 5, 'day': '2022-01-03'},
{'sum': 8, 'day': '2022-01-01'},
{'sum': 15, 'day': '2022-01-02'},
{'sum': 9, 'day': '2022-01-03'},
{'total_value': 19, 'day': '2022-01-01'},
{'total_value': 99, 'day': '2022-01-02'},
{'total_value': 15, 'day': '2022-01-03'}]
output = {}
for item in data:
if item['day'] not in output:
output[item['day']] = item
else:
output[item['day']].update(item)
print(list(output.values()))
Out:
[
{'average': 2, 'day': '2022-01-01', 'sum': 8, 'total_value': 19},
{'average': 3, 'day': '2022-01-02', 'sum': 15, 'total_value': 99},
{'average': 5, 'day': '2022-01-03', 'sum': 9, 'total_value': 15}
]
Had a bit of fun and made it with dict/list comprehension. Check out that neat | operator in python 3.9+ :-)
Python <3.9
from collections import ChainMap
data_grouped_by_day = {
day : dict(ChainMap(*[d for d in data if d["day"] == day ]))
for day in {d["day"] for d in data }
}
for day, group_data in data_grouped_by_day.items():
group_data.update(day=day)
result = list(data_grouped_by_day.values())
Python 3.9+
from collections import ChainMap
result = [
dict(ChainMap(*[d for d in data if d["day"] == day ])) | {"day" : day}
for day in {d["day"] for d in data}
]
The output in both cases is (keys order may vary)
[{'total_value': 99, 'day': '2022-01-02', 'sum': 15, 'average': 3},
{'total_value': 15, 'day': '2022-01-03', 'sum': 9, 'average': 5},
{'total_value': 19, 'day': '2022-01-01', 'sum': 8, 'average': 2}]
Related
Out of all the months in the year, I need to code the month with largest total balance (it's June as all together June has the biggest "amount" value)
lst = [
{'account': 'x\\*', 'amount': 300, 'day': 3, 'month': 'June'},
{'account': 'y\\*', 'amount': 550, 'day': 9, 'month': 'May'},
{'account': 'z\\*', 'amount': -200, 'day': 21, 'month': 'June'},
{'account': 'g', 'amount': 80, 'day': 10, 'month': 'May'},
{'account': 'x\\*', 'amount': 30, 'day': 16, 'month': 'August'},
{'account': 'x\\*', 'amount': 100, 'day': 5, 'month': 'June'},
]
The problem is that both "amount" and the name of the months are values.
I tried to find the total for each month, but I need to use for loop to code the highest month "amount".
My attempt:
get_sum = lambda my_dict, month: sum(d['amount']
for d in my_list if d['month'] == month)
total_June = get_sum(my_list,'June')
total_August = get_sum(my_list),'August')
A simple solution with pandas.
import pandas as pd
lst = [
{'account': 'x\\*', 'amount': 300, 'day': 3, 'month': 'June'},
{'account': 'y\\*', 'amount': 550, 'day': 9, 'month': 'May'},
{'account': 'z\\*', 'amount': -200, 'day': 21, 'month': 'June'},
{'account': 'g', 'amount': 80, 'day': 10, 'month': 'May'},
{'account': 'x\\*', 'amount': 30, 'day': 16, 'month': 'August'},
{'account': 'x\\*', 'amount': 100, 'day': 5, 'month': 'June'},
]
# convert list of dictionaries to dataframe
df = pd.DataFrame(lst)
# Get the row / series that has max amount.
# idxmax returns an index for loc.
max_series_by_amount = df.loc[df['amount'].idxmax(axis="index")]
# Get only month and amount in a plain list
print(max_series_by_amount[["month", "amount"]].tolist())
['May', 550]
Please note that using pandas adds a substantial amount of dependencies to the project, that said, pandas is commonly imported anyway for data science or data manipulation tasks. Pierre D solutions here are definitively faster.
One possibility (among many):
from itertools import groupby
from operator import itemgetter
mo_total = {
k: sum([d.get('amount', 0) for d in v])
for k, v in groupby(sorted(lst, key=itemgetter('month')), key=itemgetter('month'))
}
>>> mo_total
{'August': 30, 'June': 200, 'May': 630}
>>> max(mo_total.items(), key=lambda kv: kv[1])
('May', 630)
Without itemgetter:
bymonth = lambda d: d.get('month')
mo_total = {
k: sum([d.get('amount', 0) for d in v])
for k, v in groupby(sorted(lst, key=bymonth), key=bymonth)
}
Yet another way, using defaultdict:
from collections import defaultdict
tot = defaultdict(int)
for d in lst:
tot[d['month']] += d.get('amount', 0)
>>> tot
defaultdict(int, {'June': 200, 'May': 630, 'August': 30})
>>> max(tot, key=lambda k: tot[k])
'May'
I have two lists (with dicts in it):
old_device_data_list = [{'_id': ObjectId('5f48c8e34545fac49fbff5'), 'device_id': 5, 'time': datetime.datetime(2020, 8, 26, 9, 5, 39, 827000), 'values': {'count': 100, 'late': 0, 'max': 0, 'min': 0, 'on_time': 100, 'sum': 100}}]
result = [{'_id': ObjectId('5f48c8e3997640fac49fbff5'), 'device_id': 5, 'time': datetime.datetime(2020, 8, 26, 9, 5, 39, 827000), 'values': {'count': 100, 'late': 0, 'max': 0, 'min': 0, 'on_time': 100, 'sum': 100}}, {'_id': ObjectId('5f48c8e3997640fac49fbff6'), 'device_id': 4, 'time': datetime.datetime(2020, 8, 26, 9, 5, 39, 827000), 'values': {'count': 180, 'late': 0, 'max': 0, 'min': 0, 'on_time': 180, 'sum': 180}}, {'_id': ObjectId('5f48c8e3997640fac49fbff8'), 'device_id': 3, 'time': datetime.datetime(2020, 8, 27, 9, 5, 39, 827000), 'values': {'count': 50, 'late': 0, 'max': 0, 'min': 0, 'on_time': 50, 'sum': 50}}, {'_id': ObjectId('5f48c8e3997640fac49fbff7'), 'device_id': 4, 'time': datetime.datetime(2020, 8, 27, 9, 5, 39, 827000), 'values': {'count': 120, 'late': 0, 'max': 0, 'min': 0, 'on_time': 120, 'sum': 120}}, {'_id': ObjectId('5f48c8e3997640fac49fbff9'), 'device_id': 3, 'time': datetime.datetime(2020, 8, 28, 9, 5, 39, 827000), 'values': {'count': 210, 'late': 0, 'max': 0, 'min': 0, 'on_time': 210, 'sum': 210}}]
I want to delete the dicts from the old_device_data_list out of the result list. I tried it with numpy with:
numpy.setdiff1d(result, old_device_data_list)
Then I got error:
TypeError: '<' not supported between instances of 'dict' and 'dict'
The description of numpy.setdiff1d says:
Return the sorted, unique values in ar1 that are not in ar2.
In order to sort the values, it needs to compare them using the < operator. But dictionaries cannot be compared like this. The relation "smaller than" is not defined for dictionaries.
NumPy is designed for working with numeric values, not for arbitrary Python data structures.
You could use a simple list comprehension to create a list of those dictionaries that are in result but not in old_device_data_list:
result = [d for d in result if d not in old_device_data_list]
I would like to sort this list of dicts by a list key and then by date.
I am trying to sort the dicts by 'label' according the label_order and then by descending 'date'.
label_order = [3, 4, 2, 1]
data = [
{'label': 1, 'data': 5, 'date': datetime(2018, 12, 31)},
{'label': 3, 'data': 2, 'date': datetime(2017, 12, 31)},
{'label': 3, 'data': 1, 'date': datetime(2018, 12, 31)},
{'label': 4, 'data': 3, 'date': datetime(2018, 12, 31)},
{'label': 4, 'data': 4, 'date': datetime(2018, 12, 25)},
]
After sorting would look like this:
data = [
{'label': 3, 'data': 1, 'date': datetime(2018, 12, 31)},
{'label': 3, 'data': 2, 'date': datetime(2017, 12, 31)},
{'label': 4, 'data': 3, 'date': datetime(2018, 12, 31)},
{'label': 4, 'data': 4, 'date': datetime(2018, 12, 25)},
{'label': 1, 'data': 5, 'date': datetime(2018, 12, 31)},
]
I've tried lambda expressions and itemgetter, but I am having difficulty combining the right strategies for the sort key. Maybe it is just trying to do too much at one time.
Any help or direction would be appreciated.
A more efficient approach is to build a dict that maps items in label_order to indices, so that you can use the indices as keys when performing the sort:
keys = {n: i for i, n in enumerate(label_order)}
sorted(data, key=lambda d: (-keys[d['label']], d['date']), reverse=True)
This returns:
[{'label': 3, 'data': 1, 'date': datetime(2018, 12, 31)},
{'label': 3, 'data': 2, 'date': datetime(2017, 12, 31)},
{'label': 4, 'data': 3, 'date': datetime(2018, 12, 31)},
{'label': 4, 'data': 4, 'date': datetime(2018, 12, 25)},
{'label': 1, 'data': 5, 'date': datetime(2018, 12, 31)}]
It's a little tricky to sort dates in reverse order. Instead, let's use the negative of the label's index so they're sorted in descending order. Then we can reverse the sorting and get the results in the order we actually want!
from datetime import datetime
label_order = [3, 4, 2, 1]
data = [
{'label': 1, 'data': 5, 'date': datetime(2018, 12, 31)},
{'label': 3, 'data': 2, 'date': datetime(2017, 12, 31)},
{'label': 3, 'data': 1, 'date': datetime(2018, 12, 31)},
{'label': 4, 'data': 3, 'date': datetime(2018, 12, 31)},
{'label': 4, 'data': 4, 'date': datetime(2018, 12, 25)},
]
def descending_sort_key(item):
return -label_order.index(item['label']), item['date']
data.sort(key=descending_sort_key, reverse=True)
Voila - no date math or other trickery.
I have a django view that I need to query from different models and combine them, and then organize by date ('created_at'), right now when combining the models I get a list of dicts like below. How can I sort this by date.
[{'content': u'Just another another message', 'created_at':
datetime.datetime(2018, 4, 22, 15, 35, 11, 577175, tzinfo=<UTC>),
u'successmatch_id': 5, u'id': 8, 'reciever': u'UserA'},
{'content': u'testing blah', 'created_at': datetime.datetime(2018, 4,
22, 15, 33, 28, 84469, tzinfo=<UTC>), u'successmatch_id': 5, u'id': 7,
'reciever': u'UserB'}, {'content': u'Hi how are you',
'created_at': datetime.datetime(2018, 4, 22, 13, 29, 49, 516701,
tzinfo=<UTC>), u'successmatch_id': 5, u'id': 6, 'reciever':
u'UserA'}]
Python's built-in sorting has the ability to specify what metric to sort by:
x = [{"test": 1}, {"test": 2}, {"test": 0}]
x.sort(key=lambda item: item["test"])
x is edited in place, and is now:
[{'test': 0}, {'test': 1}, {'test': 2}]
So, in your case, assuming your list is called my_list, you'd want to do:
my_list.sort(key=lambda item: item["created_at"])
Or, if you wanted the newest dicts to occur first,
my_list.sort(key=lambda item: item["created_at"], reverse=True)
If you are happy using a 3rd party library, you can use pandas, which accepts a list of dictionaries.
But note that datetime objects may be converted to pandas.Timestamp objects.
import pandas as pd
import datetime
lst = [{'content': u'Just another another message',
'created_at': datetime.datetime(2018, 4, 22, 15, 35, 11, 577175, tzinfo=None),
u'successmatch_id': 5, u'id': 8, 'reciever': u'UserA'},
{'content': u'testing blah',
'created_at': datetime.datetime(2018, 4, 22, 15, 33, 28, 84469, tzinfo=None),
u'successmatch_id': 5, u'id': 7, 'reciever': u'UserB'},
{'content': u'Hi how are you',
'created_at': datetime.datetime(2018, 4, 22, 13, 29, 49, 516701, tzinfo=None),
u'successmatch_id': 5, u'id': 6, 'reciever': u'UserA'}]
res = pd.DataFrame(lst).sort_values('created_at').T.to_dict().values()
Result:
dict_values([{'content': 'Hi how are you', 'created_at': Timestamp('2018-04-22 13:29:49.516701'),
'id': 6, 'reciever': 'UserA', 'successmatch_id': 5},
{'content': 'testing blah', 'created_at': Timestamp('2018-04-22 15:33:28.084469'),
'id': 7, 'reciever': 'UserB', 'successmatch_id': 5},
{'content': 'Just another another message', 'created_at': Timestamp('2018-04-22 15:35:11.577175'),
'id': 8, 'reciever': 'UserA', 'successmatch_id': 5}])
What wrong with this code, return empty list?
week = []
for d in week:
day_num = calendar.weekday(d.year,d.month,d.day)
day_name = calendar.day_name[day_num]
daydate = { "day_name":day_name,
"day":d.day,
"month":d.month,
"year":d.year,
}
week.append(daydate)
return week
Because the list week is empty initially, the for loop is iterated zero times.
Your week list is set as [] just before the for statement, so the loop doesn't have any element to iterate on. You have to either:
remove this week = [] if week has already been declared
add elements in the list.
fixed your code. It's maybe not on week that you want to iterate but on another variable.
import calendar
from datetime import datetime
from datetime import timedelta
def generateDays(start_date,weeks):
days=7*weeks
week = []
for day in np.arange(days):
a_date = pd.to_datetime(start_date + timedelta(days=int(day)))
day_num = calendar.weekday(a_date.year,a_date.month,a_date.day)
day_name = calendar.day_name[day_num]
daydate = { "day_name":day_name,
"day":a_date.day,
"month":a_date.month,
"year":a_date.year,
}
week.append(daydate)
return week
print(generateDays(date.today(),2))
output
[{'day_name': 'Wednesday', 'day': 16, 'month': 6, 'year': 2021}, {'day_name': 'Thursday', 'day': 17, 'month': 6, 'year': 2021}, {'day_name': 'Friday', 'day': 18, 'month': 6, 'year': 2021}, {'day_name': 'Saturday', 'day': 19, 'month': 6, 'year': 2021}, {'day_name': 'Sunday', 'day': 20, 'month': 6, 'year': 2021}, {'day_name': 'Monday', 'day': 21, 'month': 6, 'year': 2021}, {'day_name': 'Tuesday', 'day': 22, 'month': 6, 'year': 2021}, {'day_name': 'Wednesday', 'day': 23, 'month': 6, 'year': 2021}, {'day_name': 'Thursday', 'day': 24, 'month': 6, 'year': 2021}, {'day_name': 'Friday', 'day': 25, 'month': 6, 'year': 2021}, {'day_name': 'Saturday', 'day': 26, 'month': 6, 'year': 2021}, {'day_name': 'Sunday', 'day': 27, 'month': 6, 'year': 2021}, {'day_name': 'Monday', 'day': 28, 'month': 6, 'year': 2021}, {'day_name': 'Tuesday', 'day': 29, 'month': 6, 'year': 2021}]