Develop Reference

How to define and implement a function that behaves like the dict constructor/update

dicts support several initialization/update arguments:
d = dict([('a',1), ('b',2)]) # list of (key,value)
d = dict({'a':1, 'b':2}) # copy of dict
d = dict(a=1, b=2) #keywords
How do I write the signature and handle the arguments of a function that handles the same set of cases?
I'm writing a class that I want to support the same set of arguments, but just store the list of (key,value) pairs in order to allow for multiple items with the same key. I'd like to be able to add to the list of pairs from dicts, or lists of pairs or keywords in a way that is the same as for dict.
The C implementation of update isn't very helpful to me as I'm implementing this in python.

Here is a function that should behave the same as dict (except for the requirement to support multiple keys):
def func(*args, **kwargs):
result = []
if len(args) > 1:
raise TypeError('expected at most 1 argument, got 2')
elif args:
if all(hasattr(args[0], a) for a in ('keys', '__getitem__')):
result.extend(dict(args[0]).items())
else:
for k, v in args[0]:
hash(k)
result.append((k, v))
result.extend(kwargs.items())
return result
A few notes:
The collections.abc.Mapping class isn't used to test for mapping objects, because dict has lower requirements than that.
The signature of the function could be defined more precisely using positional-only syntax, like this: def func(x=None, /, **kwargs). However, that requires Python >= 3.8, so a more backwards-compatible solution has been preferred.
The function should raise the same exceptions as dict, but the messages won't always be exactly the same.
Below are some simple tests that compare the behaviour of the function with the dict constructor (although it does not attempt to cover every possibility):
def test(fn):
d1 = {'a': 1, 'b': 2, 'c': 3}
d2 = {'d': 4, 'e': 5, 'f': 6}
class Maplike():
def __getitem__(self, k):
return d1[k]
def keys(self):
return d1.keys()
print('test %s\n=========\n' % fn.__name__)
print('dict:', fn(d1))
print('maplike:', fn(Maplike()))
print('seq:', fn(tuple(d1.items())))
print('it:', fn(iter(d1.items())))
print('gen:', fn(i for i in d1.items()))
print('set:', fn(set(d2.items())))
print('str:', fn(['fu', 'ba', 'r!']))
print('kwargs:', fn(**d1))
print('arg+kwargs:', fn(d1, **d2))
print('dup-keys:', fn(d1, **d1))
print('empty:', fn())
print()
try:
fn(d1, d2)
print('ERROR')
except Exception as e:
print('len-args: %s' % e)
try:
fn([(1, 2, 3)])
print('ERROR')
except Exception as e:
print('pairs: %s' % e)
try:
fn([([], 3)])
print('ERROR')
except Exception as e:
print('hashable: %s' % e)
print()
test(func)
test(dict)
Ouput:
test func
=========
dict: [('a', 1), ('b', 2), ('c', 3)]
maplike: [('a', 1), ('b', 2), ('c', 3)]
seq: [('a', 1), ('b', 2), ('c', 3)]
it: [('a', 1), ('b', 2), ('c', 3)]
gen: [('a', 1), ('b', 2), ('c', 3)]
set: [('d', 4), ('e', 5), ('f', 6)]
str: [('f', 'u'), ('b', 'a'), ('r', '!')]
kwargs: [('a', 1), ('b', 2), ('c', 3)]
arg+kwargs: [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5), ('f', 6)]
dup-keys: [('a', 1), ('b', 2), ('c', 3), ('a', 1), ('b', 2), ('c', 3)]
empty: []
len-args: expected at most 1 argument, got 2
pairs: too many values to unpack (expected 2)
hashable: unhashable type: 'list'
test dict
=========
dict: {'a': 1, 'b': 2, 'c': 3}
maplike: {'a': 1, 'b': 2, 'c': 3}
seq: {'a': 1, 'b': 2, 'c': 3}
it: {'a': 1, 'b': 2, 'c': 3}
gen: {'a': 1, 'b': 2, 'c': 3}
set: {'d': 4, 'e': 5, 'f': 6}
str: {'f': 'u', 'b': 'a', 'r': '!'}
kwargs: {'a': 1, 'b': 2, 'c': 3}
arg+kwargs: {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5, 'f': 6}
dup-keys: {'a': 1, 'b': 2, 'c': 3}
empty: {}
len-args: dict expected at most 1 argument, got 2
pairs: dictionary update sequence element #0 has length 3; 2 is required
hashable: unhashable type: 'list'

This should do the trick:
class NewDict:
def __init__(self, *args, **kwargs):
self.items = []
if kwargs:
self.items = list(kwargs.items())
elif args:
if type(args[0]) == list:
self.items = list(args[0])
elif type(args[0]) == dict:
self.items = list(args[0].items())
def __repr__(self):
s = "NewDict({"
for i, (k,v) in enumerate(self.items):
s += repr(k) + ": " + repr(v)
if i < len(self.items) - 1:
s += ", "
s += "})"
return s
d1 = NewDict()
d2 = NewDict([('a',1), ('b',2), ('a',2)])
d3 = NewDict({'a':1, 'b':2})
d4 = NewDict(a=1, b=2)
print(d1, d2, d3, d4, sep="\n")
print(d2.items)
Output:
NewDict({})
NewDict({'a': 1, 'b': 2, 'a': 2})
NewDict({'a': 1, 'b': 2})
NewDict({'a': 1, 'b': 2})
[('a', 1), ('b', 2), ('a', 2)]

I'm not saying this is a safe or great way to do this, but how about ...
def getPairs(*args, **kwargs):
pairs = []
for a in args:
if type(a) == dict:
pairs.extend([[key, val] for key, val in a.items()])
elif type(a) == list:
for a2 in a:
if len(a2) > 1:
pairs.append([a2[0], a2[1:]])
pairs.extend([[key, val] for key, val in kwargs.items()])
return pairs

How to merge 2 ordered dictionaries in python?

I have 2 ordered dictionaries, like:
a = collections.OrderedDict()
b = collections.OrderedDict()
And they have stuff in them. How do I merge these 2? I tried:
mergeDict = dict(a.items() + b.items())
but doing this it's not a ordered dictionary anymore.
What I am looking for: if a = {1, 2, 5, 6} and b = [0, 7, 3, 9} then mergeDict = {1, 2, 5, 6, 0, 7, 3, 9}

Two ways (assuming Python 3.6):
Use "update method". Suppose there are two dictionaries:
>>> d1 = collections.OrderedDict([('a', 1), ('b', 2)])
>>> d2 = {'c': 3, 'd': 4}
>>> d1.update(d2)
>>> d1
OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 4)])
Second method using 'concatenation operator (+)'
>>> d1 = collections.OrderedDict([('a', 1), ('b', 2)])
>>> d2 = {'c': 3, 'd': 4}
>>> d3 = collections.OrderedDict(list(d1.items()) + list(d2.items()))
>>> d3
OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 4)])

from itertools import chain
from collections import OrderedDict
OrderedDict(chain(a.items(), b.items()))

CPython 3.6, and any 3.7+ interpreter already preserves dictionary key ordering.
This allows you to do use the {**a, **b} syntax.
For example:
>>> a = {1: "AA", 2: "BB", 5: "CC", 6: "DD"}
>>> b = {0: "EE", 7: "FF", 3: "GG", 9: "HH"}
>>> {**a, **b}
{1: 'AA', 2: 'BB', 5: 'CC', 6: 'DD', 0: 'EE', 7: 'FF', 3: 'GG', 9: 'HH'}

instead of dict use back OrderedDict for mergeDict
mergeDict = collections.OrderedDict(a.items() + b.items())
Remark :this only works for python 2.x, add list() over dict.items() for python 3 because dict.items() no longer return list that support + operation
or use a.update(b) like #alfasin mentions in comment
i try with simple example and both method works well for me

Convert dict of nested lists to list of tuples

I have dict of nested lists:
d = {'a': [[('a1', 1, 1), ('a2', 1, 2)]], 'b': [[('b1', 2, 1), ('b2', 2, 2)]]}
print (d)
{'b': [[('b1', 2, 1), ('b2', 2, 2)]], 'a': [[('a1', 1, 1), ('a2', 1, 2)]]}
I need create list of tuples like:
[('b', 'b1', 2, 1), ('b', 'b2', 2, 2), ('a', 'a1', 1, 1), ('a', 'a2', 1, 2)]
I tried:
a = [[(k, *y) for y in v[0]] for k,v in d.items()]
a = [item for sublist in a for item in sublist]
I think my solution is a bit over-complicated. Is there some better, more pythonic, maybe one line solution?

You were almost there:
[(k, *t) for k, v in d.items() for t in v[0]]
The v[0] is needed because your values are just single-element lists with another list contained. The above can be expanded to the following nested for loops, if you wanted to figure out what it does:
for key, value in d.items(): # value is [[(...), (...), ...]]
for tup in value[0]: # each (...) from value[0]
(key, *tup) # produce a new tuple
Demo:
>>> d = {'a': [[('a1', 1, 1), ('a2', 1, 2)]], 'b': [[('b1', 2, 1), ('b2', 2, 2)]]}
>>> [(k, *t) for k, v in d.items() for t in v[0]]
[('a', 'a1', 1, 1), ('a', 'a2', 1, 2), ('b', 'b1', 2, 1), ('b', 'b2', 2, 2)]

Is this the best way to merge two dicts of dicts together and sum similar values

Is there a faster or more "pythonic" way to achieve that:
dicta = {'a':{'a1':1, 'a2':2}, 'b':{'b1': 1, 'b2': 2}}
dictb = {'b':{'b1':1, 'a2':2}, 'c':{'c1': 1, 'c2': 2}}
dictc = {}
dictc.update(dicta)
for outside_key in dictb:
if outside_key in dictc:
for inside_key in dictb[outside_key]:
if inside_key in dictc[outside_key]:
dictc[outside_key][inside_key] += dictb[outside_key][inside_key]
else:
dictc[outside_key][inside_key] = dictb[outside_key][inside_key]
else:
dictc[outside_key] = dictb[outside_key]
dictc now contains the following:
{'a': {'a1': 1, 'a2': 2},
'c': {'c2': 2, 'c1': 1},
'b': {'a2': 2, 'b1': 2, 'b2': 2}}
I'm using this in a django model definition, the above seemed the most obvious way to describe the problem but I then realised that it was not specific enough.
Here is how the django code looks like:
def get_capabilities(self):
capabilities_dict = {}
capabilities_list = ('card', 'module') #Two ManyToMany fields
capabilities_dict.update(self.barebone.get_capabilities())
# self.barebone.get_capabilities() is the only foreingkey field
# and serves as the base on which we build the capabilities list.
for capability in capabilities_list:
instances = getattr(self, capability).all()
for instance in instances:
capabilities = instance.get_capabilities()
for capability_name in capabilities:
if capability_name in capabilities_dict:
for obj in capabilities[capability_name]:
if obj in capabilities_dict[capability_name]:
capabilities_dict[capability_name][obj] += capabilities[capability_name][obj]
else:
capabilities_dict[capability_name][obj] = capabilities[capability_name][obj]
else:
capabilities_dict[capability_name] = capabilities[capability_name]
return capabilities_dict
self.barebone.get_capabilities() looks like this:
{'compatible_bus_types': {<BusType: PCI-X>: 1},
'compatible_storage_interfaces': {<StorageInterface: SATA>: 8},
'compatible_storage_form_factors': {<StorageFormFactor: 3.5">: 4}}
and the function get_capabilities() above returns this:
{'compatible_network_connectors': {},
'compatible_storage_interfaces': {<StorageInterface: SATA>: 8,
<StorageInterface: SAS>: 8},
'compatible_network_standards': {},
'compatible_storage_form_factors': {<StorageFormFactor: 3.5">: 4},
'compatible_bus_types': {<BusType: PCI-X>: 1},
'compatible_network_form_factors': {}}
Each <> enclosed "inner-keys" are in fact another model instance.
Thanks,

Using collections.Counter with a normal dict:
>>> from collections import Counter
>>> dictc = {}
>>> for d in (dicta, dictb):
for k, v in d.items():
dictc[k] = dictc.get(k, Counter()) + Counter(v)
...
>>> dictc
{'a': Counter({'a2': 2, 'a1': 1}),
'c': Counter({'c2': 2, 'c1': 1}),
'b': Counter({'a2': 2, 'b1': 2, 'b2': 2})}
or with a defaultdict:
>>> from collections import Counter, defaultdict
>>> dictc = defaultdict(Counter)
>>> for d in (dicta, dictb):
for k, v in d.items():
dictc[k] += Counter(v)
...
>>> dictc
defaultdict(<class 'collections.Counter'>,
{'a': Counter({'a2': 2, 'a1': 1}),
'c': Counter({'c2': 2, 'c1': 1}),
'b': Counter({'a2': 2, 'b1': 2, 'b2': 2})})

You use nested dicts but I think there is an easier data-structure: You can take a tuple like ('a','a2') as key instead. Your dicts would look like this:
dicta = {('a', 'a1'): 1, ('a', 'a2'): 2, ('b', 'b1'): 1, ('b', 'b2'): 2}
dictb = {('b', 'a2'): 2, ('b', 'b1'): 1, ('c', 'c1'): 1, ('c', 'c2'): 2}
Accessing the values would work like this:
print dicta['a', 'a1']
Out: 1
instead of
print dicta['a']['a1']
Out: 1
To change your data into that structure you can you this function:
def unnestdict(dct):
return { (key1, key2):value
for key1, sub_dct in dct.iteritems()
for key2, value in sub_dct.iteritems()}
If you use this data structure you can easily use the Counter-class from collections-module:
from collections import Counter
counterC = Counter(dicta) + Counter(dictb)
print counterC
Out: Counter({('b', 'b2'): 2, ('a', 'a2'): 2, ('b', 'b1'): 2, ('b', 'a2'): 2, ('c', 'c2'): 2, ('a', 'a1'): 1, ('c', 'c1'): 1})
dictc = dict(counterC)
print dictc
Out: {('b', 'a2'): 2, ('b', 'b1'): 1, ('c', 'c1'): 1, ('c', 'c2'): 2}

One-step initialization of defaultdict that appends to list?

It would be convenient if a defaultdict could be initialized along the following lines
d = defaultdict(list, (('a', 1), ('b', 2), ('c', 3), ('d', 4), ('a', 2),
('b', 3)))
to produce
defaultdict(<type 'list'>, {'a': [1, 2], 'c': [3], 'b': [2, 3], 'd': [4]})
Instead, I get
defaultdict(<type 'list'>, {'a': 2, 'c': 3, 'b': 3, 'd': 4})
To get what I need, I end up having to do this:
d = defaultdict(list)
for x, y in (('a', 1), ('b', 2), ('c', 3), ('d', 4), ('a', 2), ('b', 3)):
d[x].append(y)
This is IMO one step more than should be necessary, am I missing something here?

What you're apparently missing is that defaultdict is a straightforward (not especially "magical") subclass of dict. All the first argument does is provide a factory function for missing keys. When you initialize a defaultdict, you're initializing a dict.
If you want to produce
defaultdict(<type 'list'>, {'a': [1, 2], 'c': [3], 'b': [2, 3], 'd': [4]})
you should be initializing it the way you would initialize any other dict whose values are lists:
d = defaultdict(list, (('a', [1, 2]), ('b', [2, 3]), ('c', [3]), ('d', [4])))
If your initial data has to be in the form of tuples whose 2nd element is always an integer, then just go with the for loop. You call it one extra step; I call it the clear and obvious way to do it.

the behavior you describe would not be consistent with the defaultdicts other behaviors. Seems like what you want is FooDict such that
>>> f = FooDict()
>>> f['a'] = 1
>>> f['a'] = 2
>>> f['a']
[1, 2]
We can do that, but not with defaultdict; lets call it AppendDict
import collections
class AppendDict(collections.MutableMapping):
def __init__(self, container=list, append=None, pairs=()):
self.container = collections.defaultdict(container)
self.append = append or list.append
for key, value in pairs:
self[key] = value
def __setitem__(self, key, value):
self.append(self.container[key], value)
def __getitem__(self, key): return self.container[key]
def __delitem__(self, key): del self.container[key]
def __iter__(self): return iter(self.container)
def __len__(self): return len(self.container)

Sorting and itertools.groupby go a long way:
>>> L = [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('a', 2), ('b', 3)]
>>> L.sort(key=lambda t:t[0])
>>> d = defaultdict(list, [(tup[0], [t[1] for t in tup[1]]) for tup in itertools.groupby(L, key=lambda t: t[0])])
>>> d
defaultdict(<type 'list'>, {'a': [1, 2], 'c': [3], 'b': [2, 3], 'd': [4]})
To make this more of a one-liner:
L = [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('a', 2), ('b', 3)]
d = defaultdict(list, [(tup[0], [t[1] for t in tup[1]]) for tup in itertools.groupby(sorted(L, key=operator.itemgetter(0)), key=lambda t: t[0])])
Hope this helps

I think most of this is a lot of smoke and mirrors to avoid a simple for loop:
di={}
for k,v in [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('a', 2),('b', 3)]:
di.setdefault(k,[]).append(v)
# di={'a': [1, 2], 'c': [3], 'b': [2, 3], 'd': [4]}
If your goal is one line and you want abusive syntax that I cannot at all endorse or support you can use a side effect comprehension:
>>> li=[('a', 1), ('b', 2), ('c', 3), ('d', 4), ('a', 2),('b', 3)]
>>> di={};{di.setdefault(k[0],[]).append(k[1]) for k in li}
set([None])
>>> di
{'a': [1, 2], 'c': [3], 'b': [2, 3], 'd': [4]}
If you really want to go overboard into the unreadable:
>>> {k1:[e for _,e in v1] for k1,v1 in {k:filter(lambda x: x[0]==k,li) for k,v in li}.items()}
{'a': [1, 2], 'c': [3], 'b': [2, 3], 'd': [4]}
You don't want to do that. Use the for loop Luke!

>>> kvs = [(1,2), (2,3), (1,3)]
>>> reduce(
... lambda d,(k,v): d[k].append(v) or d,
... kvs,
... defaultdict(list))
defaultdict(<type 'list'>, {1: [2, 3], 2: [3]})