Develop Reference

Python create list of dicts

I am new to python and I am trying to construct data structure from existing data.
I have following:
[
{'UserName': 'aaa', 'AccessKeyId': 'AKIAYWQTISJD6X27YVK', 'Status': 'Active', 'CreateDate': datetime.datetime(2022, 9, 8, 15, 56, 39, tzinfo=tzutc())},
{'UserName': 'eee', 'AccessKeyId': 'AKIAYWQTISJD6QXMAKY', 'Status': 'Active', 'CreateDate': datetime.datetime(2023, 1, 24, 12, 30, 59, tzinfo=tzutc())},
{'UserName': 'eee', 'AccessKeyId': 'AKIAYWQTISJDUARK6FV', 'Status': 'Active', 'CreateDate': datetime.datetime(2023, 1, 24, 16, 58, 38, tzinfo=tzutc())}
]
I need to get this:
{
"aaa": [
{'AccessKeyId': 'AKIAYWQTISJD6X27YVK', 'Status': 'Active', 'CreateDate': datetime.datetime(2022, 9, 8, 15, 56, 39, tzinfo=tzutc())}],
"eee": [
{'AccessKeyId': 'AKIAYWQTISJD6QXMAKY', 'Status': 'Active', 'CreateDate': datetime.datetime(2023, 1, 24, 12, 30, 59, tzinfo=tzutc())},
{'AccessKeyId': 'AKIAYWQTISJDUARK6FV', 'Status': 'Active', 'CreateDate': datetime.datetime(2023, 1, 24, 16, 58, 38, tzinfo=tzutc())}
]
}
I tried following:
list_per_user = {i['UserName']: copy.deepcopy(i) for i in key_list}
for obj in list_per_user:
del list_per_user[obj]['UserName']
but I am missing array here. So in case of two keys per user I will have only last one with this. I don't know how to get the list I need per user.
Thanks!

Create an external dict that maps username -> list of entries.
data = [
{'UserName': 'aaa', 'AccessKeyId': 'AKIAYWQTISJD6X27YVK', 'Status': 'Active', 'CreateDate': datetime.datetime(2022, 9, 8, 15, 56, 39, tzinfo=tzutc())},
{'UserName': 'eee', 'AccessKeyId': 'AKIAYWQTISJD6QXMAKY', 'Status': 'Active', 'CreateDate': datetime.datetime(2023, 1, 24, 12, 30, 59, tzinfo=tzutc())},
{'UserName': 'eee', 'AccessKeyId': 'AKIAYWQTISJDUARK6FV', 'Status': 'Active', 'CreateDate': datetime.datetime(2023, 1, 24, 16, 58, 38, tzinfo=tzutc())}
]
new_data = {}
for entry in data:
new_data.setdefault(entry["UserName"], []).append(
{k: v for k, v in entry.items() if k != "UserName"}
)
print(new_data)
Output (some fields hidden because I don't want to import those libraries in my repl, but they'll be there when you run it)
{'aaa': [{'AccessKeyId': 'AKIAYWQTISJD6X27YVK', 'Status': 'Active'}],
'eee': [{'AccessKeyId': 'AKIAYWQTISJD6QXMAKY', 'Status': 'Active'},
{'AccessKeyId': 'AKIAYWQTISJDUARK6FV', 'Status': 'Active'}]}

How to calculate cumulative sum with django ORM?

I'm trying to group_by() data based on dates and with every day I want to calculate Count on that day also the total count so far.
Sample output I'm getting:
[
{
"dates": "2022-11-07",
"count": 1
},
{
"dates": "2022-11-08",
"count": 3
},
{
"dates": "2022-11-09",
"count": 33
}
]
Sample output I'm trying to achieve:
[
{
"dates": "2022-11-07",
"count": 1,
"cumulative_count": 1
},
{
"dates": "2022-11-08",
"count": 3,
"cumulative_count": 4
},
{
"dates": "2022-11-09",
"count": 33,
"cumulative_count": 37
}
]
Here's my query:
self.serializer_class.Meta.model.objects.all().annotate(dates=TruncDate("date__date")).values("dates").order_by("dates").annotate(count=Count("channel", distinct=True)).values("count", "dates")
How can I extend this query to get a cumulative sum as well?

I tried to solve your problem like this
models.py
class Demo(models.Model):
count =models.IntegerField()
dates = models.DateField()
serializers.py
class DemoSerializer(serializers.ModelSerializer):
class Meta:
model = Demo
fields = "__all__"
Views.py
class DemoAPI(APIView):
def get(self, request, pk=None, format=None):
data = Demo.objects.all()
cumulative_count= 0
# Normal Django ORM Queruset
print('--------- Default Queryset Response ---------')
for i in data:
del i.__dict__['_state']
print(i.__dict__)
# Adding cumulative_count key in ORM Queryset
for i in data:
cumulative_count += i.__dict__['count']
i.__dict__['cumulative_count'] = cumulative_count
# Updated Django ORM Queruset with cumulative_count
print('--------- Updated Queryset Response ---------')
for i in data:
# del i.__dict__['_state']
print(i.__dict__)
Output before delete _state key from Queryset
#--------- Default Queryset Response ---------
{'_state': <django.db.models.base.ModelState object at 0x000001A07002A680>, 'id': 1, 'count': 1, 'dates': datetime.date(2022, 11, 7)}
{'_state': <django.db.models.base.ModelState object at 0x000001A07002A5C0>, 'id': 2, 'count': 3, 'dates': datetime.date(2022, 11, 8)}
{'_state': <django.db.models.base.ModelState object at 0x000001A07002A7A0>, 'id': 3, 'count': 33, 'dates': datetime.date(2022, 11, 9)}
#--------- Updated Queryset Response ---------
{'_state': <django.db.models.base.ModelState object at 0x000002DAB66E0AC0>, 'id': 1, 'count': 1, 'dates': datetime.date(2022, 11, 7), 'cumulative_count': 1}
{'_state': <django.db.models.base.ModelState object at 0x000002DAB66E0C10>, 'id': 2, 'count': 3, 'dates': datetime.date(2022, 11, 8), 'cumulative_count': 4}
{'_state': <django.db.models.base.ModelState object at 0x000002DAB66E0D60>, 'id': 3, 'count': 33, 'dates': datetime.date(2022, 11, 9), 'cumulative_count': 37}
Output after delete _state key from Queryset Added cumulative_count key in Queryset
#--------- Default Queryset Response ---------
{'id': 1, 'count': 1, 'dates': datetime.date(2022, 11, 7)}
{'id': 2, 'count': 3, 'dates': datetime.date(2022, 11, 8)}
{'id': 3, 'count': 33, 'dates': datetime.date(2022, 11, 9)}
#--------- Updated Queryset Response ---------
{'id': 1, 'count': 1, 'dates': datetime.date(2022, 11, 7), 'cumulative_count': 1}
{'id': 2, 'count': 3, 'dates': datetime.date(2022, 11, 8), 'cumulative_count': 4}
{'id': 3, 'count': 33, 'dates': datetime.date(2022, 11, 9), 'cumulative_count': 37}

delete dict in list from dict in list

I have two lists (with dicts in it):
old_device_data_list = [{'_id': ObjectId('5f48c8e34545fac49fbff5'), 'device_id': 5, 'time': datetime.datetime(2020, 8, 26, 9, 5, 39, 827000), 'values': {'count': 100, 'late': 0, 'max': 0, 'min': 0, 'on_time': 100, 'sum': 100}}]
result = [{'_id': ObjectId('5f48c8e3997640fac49fbff5'), 'device_id': 5, 'time': datetime.datetime(2020, 8, 26, 9, 5, 39, 827000), 'values': {'count': 100, 'late': 0, 'max': 0, 'min': 0, 'on_time': 100, 'sum': 100}}, {'_id': ObjectId('5f48c8e3997640fac49fbff6'), 'device_id': 4, 'time': datetime.datetime(2020, 8, 26, 9, 5, 39, 827000), 'values': {'count': 180, 'late': 0, 'max': 0, 'min': 0, 'on_time': 180, 'sum': 180}}, {'_id': ObjectId('5f48c8e3997640fac49fbff8'), 'device_id': 3, 'time': datetime.datetime(2020, 8, 27, 9, 5, 39, 827000), 'values': {'count': 50, 'late': 0, 'max': 0, 'min': 0, 'on_time': 50, 'sum': 50}}, {'_id': ObjectId('5f48c8e3997640fac49fbff7'), 'device_id': 4, 'time': datetime.datetime(2020, 8, 27, 9, 5, 39, 827000), 'values': {'count': 120, 'late': 0, 'max': 0, 'min': 0, 'on_time': 120, 'sum': 120}}, {'_id': ObjectId('5f48c8e3997640fac49fbff9'), 'device_id': 3, 'time': datetime.datetime(2020, 8, 28, 9, 5, 39, 827000), 'values': {'count': 210, 'late': 0, 'max': 0, 'min': 0, 'on_time': 210, 'sum': 210}}]
I want to delete the dicts from the old_device_data_list out of the result list. I tried it with numpy with:
numpy.setdiff1d(result, old_device_data_list)
Then I got error:
TypeError: '<' not supported between instances of 'dict' and 'dict'

The description of numpy.setdiff1d says:
Return the sorted, unique values in ar1 that are not in ar2.
In order to sort the values, it needs to compare them using the < operator. But dictionaries cannot be compared like this. The relation "smaller than" is not defined for dictionaries.
NumPy is designed for working with numeric values, not for arbitrary Python data structures.
You could use a simple list comprehension to create a list of those dictionaries that are in result but not in old_device_data_list:
result = [d for d in result if d not in old_device_data_list]

Merging overlapping interval objects with dependencies

i need to merge interval objects to get distinct ranges of intervals based on extra parameters. How is the best way to do that?
It's about unambiguous statement whether in a given hour state is true. The returned list must have non-duplicated intervals.
Interval object description:
{
'startDate': datetime.datetime, # start of interval
'endDate': datetime.datetime, # end of interval
'prioritized': bool # if True - it's always important, override no-prioritized intervals
'state': bool # result of interval
}
In the examples below i changed startDate/endDate to strings to make them look better.
Interval list look like:
interval_list = [
{'startDate': '10:00:00', 'endDate': '12:00:00', 'prioritized': False, 'state': False},
{'startDate': '11:00:00', 'endDate': '18:00:00', 'prioritized': True, 'state': True},
{'startDate': '13:00:00', 'endDate': '17:00:00', 'prioritized': False, 'state': False},
{'startDate': '17:00:00', 'endDate': '20:00:00', 'prioritized': False, 'state': True},
{'startDate': '19:30:00', 'endDate': '19:45:00', 'prioritized': True, 'state': False}
]
I am trying to achieve the following:
merge(interval_list) should return:
[
{'startDate': '10:00:00', 'endDate': '11:00:00', 'state': False},
{'startDate': '11:00:00', 'endDate': '19:30:00', 'state': True},
{'startDate': '19:30:00', 'endDate': '19:45:00', 'state': False},
{'startDate': '19:45:00', 'endDate': '20:00:00', 'state': True},
]
I have following not completed code right now:
def merge_range(ranges: list):
ranges = sorted(ranges, key=lambda x: x['startDate'])
last_interval = dict(ranges[0])
for current_interval in sorted(ranges, key=lambda x: x['startDate']):
if current_interval['startDate'] > last_interval['endDate']:
yield dict(last_interval)
last_interval['startDate'] = current_interval['startDate']
last_interval['endDate'] = current_interval['endDate']
last_interval['prioritized'] = current_interval['prioritized']
last_interval['state'] = current_interval['state']
else:
if current_interval['state'] == last_interval['state']:
last_interval['endDate'] = max(last_interval['endDate'], current_interval['endDate'])
else:
pass # i stopped here
yield dict(last_interval)
And use it by merged_interval_list = list(merge_range(interval_list))
Is it a good way ?

I got an answer for this question:
For first i separate events to prioritized and non-prioritize lists.
Based on the priority list, I create a negation of the interval on a given day.
Next i set prioritized list as main list and start iterate over non-prioritize list.
import datetime
from pprint import pprint
df = "%Y-%m-%d %H:%M:%S"
ds = "%Y-%m-%d"
events = {}
prioritized_events = {}
events["2019-05-10"] = [{
'startDate': datetime.datetime.strptime("2019-05-10 01:00:00", df),
'endDate': datetime.datetime.strptime("2019-05-10 02:00:00", df),
'state': True
}, {
'startDate': datetime.datetime.strptime("2019-05-10 10:00:00", df),
'endDate': datetime.datetime.strptime("2019-05-10 12:00:00", df),
'state': False
}, {
'startDate': datetime.datetime.strptime("2019-05-10 13:00:00", df),
'endDate': datetime.datetime.strptime("2019-05-10 17:00:00", df),
'state': False
}, {
'startDate': datetime.datetime.strptime("2019-05-10 17:00:00", df),
'endDate': datetime.datetime.strptime("2019-05-10 20:00:00", df),
'state': True
}]
prioritized_events["2019-05-10"] = [{
'startDate': datetime.datetime.strptime("2019-05-10 11:00:00", df),
'endDate': datetime.datetime.strptime("2019-05-10 18:00:00", df),
'state': True
}, {
'startDate': datetime.datetime.strptime("2019-05-10 19:30:00", df),
'endDate': datetime.datetime.strptime("2019-05-10 20:00:00", df),
'state': False
}]
allowed_intervals = []
for event_date in prioritized_events:
minimal_time = datetime.datetime.combine(datetime.datetime.strptime(event_date, ds), datetime.time.min)
maximum_time = datetime.datetime.combine(datetime.datetime.strptime(event_date, ds), datetime.time.max)
for ev in prioritized_events[event_date]:
if ev['startDate'] != minimal_time:
allowed_intervals.append({
'startDate': minimal_time,
'endDate': ev['startDate']
})
minimal_time = ev['endDate']
if prioritized_events[event_date][len(prioritized_events[event_date]) - 1]['endDate'] != maximum_time:
allowed_intervals.append({
'startDate': prioritized_events[event_date][len(prioritized_events[event_date]) - 1]['endDate'],
'endDate': maximum_time
})
for event_date in events:
if event_date not in prioritized_events:
prioritized_events[event_date] = events[event_date]
else:
for ev in events[event_date]:
start = ev['startDate']
end = ev['endDate']
state = ev['state']
done = False
for allowed_interval in allowed_intervals:
if start >= allowed_interval['startDate'] and end <= allowed_interval['endDate']:
prioritized_events[event_date].append({
'startDate': start,
'endDate': end,
'state': state
})
done = True
break
elif allowed_interval['startDate'] <= start < allowed_interval['endDate'] < end:
prioritized_events[event_date].append({
'startDate': start,
'endDate': allowed_interval['endDate'],
'state': state
})
start = allowed_interval['endDate']
elif start < allowed_interval['startDate'] and start < allowed_interval['endDate'] < end:
prioritized_events[event_date].append({
'startDate': allowed_interval['startDate'],
'endDate': allowed_interval['endDate'],
'state': state
})
start = allowed_interval['endDate']
elif start < allowed_interval['startDate'] and start < allowed_interval['endDate'] and allowed_interval['startDate'] < end <= allowed_interval['endDate']:
prioritized_events[event_date].append({
'startDate': allowed_interval['startDate'],
'endDate': end,
'state': state
})
start = end
if done:
continue
prioritized_events[event_date] = sorted(prioritized_events[event_date], key=lambda k: k['startDate'])
And now sorted list:
pprint(prioritized_events["2019-05-10"])
returns:
[
{'startDate': datetime.datetime(2019, 5, 10, 1, 0),
'endDate': datetime.datetime(2019, 5, 10, 2, 0),
'state': True
},
{'startDate': datetime.datetime(2019, 5, 10, 10, 0),
'endDate': datetime.datetime(2019, 5, 10, 11, 0),
'state': False
},
{'startDate': datetime.datetime(2019, 5, 10, 11, 0),
'endDate': datetime.datetime(2019, 5, 10, 18, 0),
'state': True
},
{'startDate': datetime.datetime(2019, 5, 10, 18, 0),
'endDate': datetime.datetime(2019, 5, 10, 19, 30),
'state': True
},
{'startDate': datetime.datetime(2019, 5, 10, 19, 30),
'endDate': datetime.datetime(2019, 5, 10, 20, 0),
'state': False
}
]

When we deal with time intervals, the main idea is to sort the dates (start and end) along with their status: start or end. Here, we need an access to the original interval too, to handle priorities and states.
Let's try with this list:
interval_list = [
{'startDate': '10:00:00', 'endDate': '12:00:00', 'prioritized': False, 'state': False},
{'startDate': '11:00:00', 'endDate': '18:00:00', 'prioritized': True, 'state': True},
{'startDate': '13:00:00', 'endDate': '17:00:00', 'prioritized': False, 'state': False},
{'startDate': '17:00:00', 'endDate': '20:00:00', 'prioritized': False, 'state': True},
{'startDate': '19:30:00', 'endDate': '19:45:00', 'prioritized': True, 'state': False}
]
First, we convert datestrings to dates (as you did):
import datetime
day = '2019-05-10'
def get_datetime(d, t):
return datetime.datetime.strptime(d+" "+t, "%Y-%m-%d %H:%M:%S")
for interval in interval_list:
interval['startDate'] = get_datetime(day, interval['startDate'])
interval['endDate'] = get_datetime(day, interval['endDate'])
Now, we build a new list with the needed information:
L = sorted(
[(interval['startDate'], 1, i) for i, interval in enumerate(interval_list)]
+[(interval['endDate'], -1, i) for i, interval in enumerate(interval_list)]
)
L is the following list of tuples (date, dir, index) (dir: 1 means it's a start date, -1 means it's an end date):
[(datetime.datetime(2019, 5, 10, 10, 0), 1, 0), (datetime.datetime(2019, 5, 10, 11, 0), 1, 1), (datetime.datetime(2019, 5, 10, 12, 0), -1, 0), (datetime.datetime(2019, 5, 10, 13, 0), 1, 2), (datetime.datetime(2019, 5, 10, 17, 0), -1, 2), (datetime.datetime(2019, 5, 10, 17, 0), 1, 3), (datetime.datetime(2019, 5, 10, 18, 0), -1, 1), (datetime.datetime(2019, 5, 10, 19, 30), 1, 4), (datetime.datetime(2019, 5, 10, 19, 45), -1, 4), (datetime.datetime(2019, 5, 10, 20, 0), -1, 3)]
Now we can iterate over L and keep a track of the current state and priority to yield dates when state is modified according to given priority:
def interval_info(i):
interval = interval_list[i]
return interval['state'], interval['prioritized']
T = []
stack = []
for boundary_date, direction, i in L:
state, prioritized = interval_info(i) # state and priority of the current date
if direction == 1: # start date
if stack:
prev_state, prev_prioritized = interval_info(stack[-1]) # previous infos
if state != prev_state and prioritized >= prev_prioritized: # enter a new state with a greater or equal priority
T.append((boundary_date, state)) # enter in new state
else: # begin of covered area
T.append((boundary_date, state)) # enter in new state
stack.append(i) # add the opened interval
elif direction == -1: # end date
stack.remove(i) # remove the closed interval (i is a *value* in stack)
if stack:
prev_state, prev_prioritized = interval_info(stack[-1])
if state != prev_state and not prev_prioritized: # leave a non priority state
T.append((boundary_date, prev_state)) # re-enter in prev state
else: # end of covered area
T.append((boundary_date, None)) # enter in None state
The value of T is:
[(datetime.datetime(2019, 5, 10, 10, 0), False), (datetime.datetime(2019, 5, 10, 11, 0), True), (datetime.datetime(2019, 5, 10, 19, 30), False), (datetime.datetime(2019, 5, 10, 19, 45), True), (datetime.datetime(2019, 5, 10, 20, 0), None)]
You can then easily produce the output you wanted. Hope it helps!
EDIT: Bonus: how to convert start dates to time intervals:
>>> import datetime
>>> T = [(datetime.datetime(2019, 5, 10, 10, 0), False), (datetime.datetime(2019, 5, 10, 11, 0), True), (datetime.datetime(2019, 5, 10, 19, 30), False), (datetime.datetime(2019, 5, 10, 19, 45), True), (datetime.datetime(2019, 5, 10, 20, 0), None)]
>>> [{'startDate': s[0], 'endDate': e[0], 'state': s[1]} for s,e in zip(T, T[1:])]
[{'startDate': datetime.datetime(2019, 5, 10, 10, 0), 'endDate': datetime.datetime(2019, 5, 10, 11, 0), 'state': False}, {'startDate': datetime.datetime(2019, 5, 10, 11, 0), 'endDate': datetime.datetime(2019, 5, 10, 19, 30), 'state': True}, {'startDate': datetime.datetime(2019, 5, 10, 19, 30), 'endDate': datetime.datetime(2019, 5, 10, 19, 45), 'state': False}, {'startDate': datetime.datetime(2019, 5, 10, 19, 45), 'endDate': datetime.datetime(2019, 5, 10, 20, 0), 'state': True}]
You just have to zip every start date with the the next one to get intervals.

Trying To Sort A List Of Dicts After Combining Django Query

I have a django view that I need to query from different models and combine them, and then organize by date ('created_at'), right now when combining the models I get a list of dicts like below. How can I sort this by date.
[{'content': u'Just another another message', 'created_at':
datetime.datetime(2018, 4, 22, 15, 35, 11, 577175, tzinfo=<UTC>),
u'successmatch_id': 5, u'id': 8, 'reciever': u'UserA'},
{'content': u'testing blah', 'created_at': datetime.datetime(2018, 4,
22, 15, 33, 28, 84469, tzinfo=<UTC>), u'successmatch_id': 5, u'id': 7,
'reciever': u'UserB'}, {'content': u'Hi how are you',
'created_at': datetime.datetime(2018, 4, 22, 13, 29, 49, 516701,
tzinfo=<UTC>), u'successmatch_id': 5, u'id': 6, 'reciever':
u'UserA'}]

Python's built-in sorting has the ability to specify what metric to sort by:
x = [{"test": 1}, {"test": 2}, {"test": 0}]
x.sort(key=lambda item: item["test"])
x is edited in place, and is now:
[{'test': 0}, {'test': 1}, {'test': 2}]
So, in your case, assuming your list is called my_list, you'd want to do:
my_list.sort(key=lambda item: item["created_at"])
Or, if you wanted the newest dicts to occur first,
my_list.sort(key=lambda item: item["created_at"], reverse=True)

If you are happy using a 3rd party library, you can use pandas, which accepts a list of dictionaries.
But note that datetime objects may be converted to pandas.Timestamp objects.
import pandas as pd
import datetime
lst = [{'content': u'Just another another message',
'created_at': datetime.datetime(2018, 4, 22, 15, 35, 11, 577175, tzinfo=None),
u'successmatch_id': 5, u'id': 8, 'reciever': u'UserA'},
{'content': u'testing blah',
'created_at': datetime.datetime(2018, 4, 22, 15, 33, 28, 84469, tzinfo=None),
u'successmatch_id': 5, u'id': 7, 'reciever': u'UserB'},
{'content': u'Hi how are you',
'created_at': datetime.datetime(2018, 4, 22, 13, 29, 49, 516701, tzinfo=None),
u'successmatch_id': 5, u'id': 6, 'reciever': u'UserA'}]
res = pd.DataFrame(lst).sort_values('created_at').T.to_dict().values()
Result:
dict_values([{'content': 'Hi how are you', 'created_at': Timestamp('2018-04-22 13:29:49.516701'),
'id': 6, 'reciever': 'UserA', 'successmatch_id': 5},
{'content': 'testing blah', 'created_at': Timestamp('2018-04-22 15:33:28.084469'),
'id': 7, 'reciever': 'UserB', 'successmatch_id': 5},
{'content': 'Just another another message', 'created_at': Timestamp('2018-04-22 15:35:11.577175'),
'id': 8, 'reciever': 'UserA', 'successmatch_id': 5}])