I am playing a little bit around with Python metaprogramming.
class FormMetaClass(type):
def __new__(cls, clsname, bases, methods):
# Attach attribute names to the descriptors
for key, value in methods.items():
if isinstance(value, FieldDescriptor):
value.name = key
return type.__new__(cls, clsname, bases, methods)
class Form(metaclass=FormMetaClass):
#classmethod
def from_json(cls, incoming):
instance = cls()
data = json.loads(incoming)
for k, v in data.items():
if (not hasattr(instance, k)):
raise KeyError("Atrribute not found")
instance.__setattr__(k, v)
return cls
class MyForm(Form):
first_name = String()
last_name = String()
age = Integer()
def __repr__(self):
return "{} {}".format(self.first_name, self.last_name)
def main():
data = json.dumps({'first_name': 'Thomas',
'last_name': 'Junk'})
form = MyForm.from_json(data)
print(form)
if __name__ == "__main__":
main()
class FieldDescriptor:
def __init__(self, name=None, **opts):
self.name = name
for key, value in opts.items():
setattr(self, key, value)
def __set__(self, instance, value):
instance.__dict__[self.name] = value
class Typechecked(FieldDescriptor):
expected_type = type(None)
def __set__(self, instance, value):
if not isinstance(value, self.expected_type):
raise TypeError('expected ' + str(self.expected_type))
super().__set__(instance, value)
class Integer(Typechecked):
expected_type = int
class String(Typechecked):
expected_type = str
I have a Form which has a metaclass FormMetaClass.
To have an alternative constructor I am using a #classmethod.
I create an instance, which seem to work so far.
What doesn't work is calling the __repr__ (or __str__ interchangeably).
When I create an instance via MyForm() everything is fine.
When I create an instance via the #classmethod, some "default" implementation is taken.
I expected Thomas Junk, but I get <class '__main__.MyForm'>
Could you give me a hint, what I am overlooking?
You are returning the class, not the newly created instance:
return cls
So you return MyForm, not the new instance MyForm() you just set all the attributes on. And you indeed see the repr() output for the class:
>>> form is MyForm
True
>>> print(MyForm)
<class '__main__.MyForm'>
The fix is simple, return instance instead:
return instance
or, as a full method:
#classmethod
def from_json(cls, incoming):
instance = cls()
data = json.loads(incoming)
for k, v in data.items():
if (not hasattr(instance, k)):
raise KeyError("Atrribute not found")
instance.__setattr__(k, v)
return instance
at which point the method returns an instance and everything works:
>>> isinstance(form, MyForm)
True
>>> print(form)
Thomas Junk
Related
Lets take the following example of class decorators (origin http://www.informit.com/articles/article.aspx?p=1309289&seqNum=4):
class GenericDescriptor:
def __init__(self, getter, setter):
self.getter = getter
self.setter = setter
def __get__(self, instance, owner=None):
if instance is None:
return self
return self.getter(instance)
def __set__(self, instance, value):
return self.setter(instance, value)
def valid_string(attr_name, empty_allowed=True, regex=None,
acceptable=None):
def decorator(cls):
name = "__" + attr_name
def getter(self):
return getattr(self, name)
def setter(self, value):
assert isinstance(value, str), (attr_name +
" must be a string")
if not empty_allowed and not value:
raise ValueError("{0} may not be empty".format(
attr_name))
if ((acceptable is not None and value not in acceptable) or
(regex is not None and not regex.match(value))):
raise ValueError("{attr_name} cannot be set to "
"{value}".format(**locals()))
setattr(self, name, value)
setattr(cls, attr_name, GenericDescriptor(getter, setter))
return cls
return decorator
#valid_string("name", empty_allowed=False)
class StockItem:
name = None
def __init__(self, **kwargs):
if kwargs.get('second_call'):
pass
# proceed normally without calling #valid_string
self.name = kwargs.get('name', None)
self.price = kwargs.get('price', None)
self.quantity = kwargs.get('quantity', None)
if __name__ == "__main__":
import doctest
doctest.testmod()
# valid value for name
cameras1 = StockItem(name="Camera", price=45.99, quatity=2)
# invalid value for name according to #valid_string but I need this to be also valid if 'second_call'
cameras2 = StockItem(name=67, price=45.99, quatity=2, second_call=True)
The StockItem class constructor is invoked twice and on the second turn I want the #valid_string decorator to be somehow canceled (I don't want name attribute's value to be altered anymore).
Read the Python Cookbook and saw descriptors, particularly the example for enforcing types when using class attributes. I am writing a few classes where that would be useful, but I would also like to enforce immutability. How to do it? Type checking descriptor adapted from the book:
class Descriptor(object):
def __init__(self, name=None, **kwargs):
self.name = name
for key, value in kwargs.items():
setattr(self, key, value)
def __set__(self, instance, value):
instance.__dict__[self.name] = value
# by default allows None
class Typed(Descriptor):
def __init__(self, expected_types=None, **kwargs):
self.expected_types = expected_types
super().__init__(**kwargs)
def __set__(self, instance, value):
if value is not None and not isinstance(value, self.expected_types):
raise TypeError('Expected: {}'.format(str(self.expected_types)))
super(Typed, self).__set__(instance, value)
class T(object):
v = Typed(int)
def __init__(self, v):
self.v = v
Attempt #1: add a self.is_set attribute to Typed
# by default allows None
class ImmutableTyped(Descriptor):
def __init__(self, expected_types=None, **kwargs):
self.expected_types = expected_types
self.is_set = False
super().__init__(**kwargs)
def __set__(self, instance, value):
if self.is_set:
raise ImmutableException(...)
if value is not None and not isinstance(value, self.expected_types):
raise TypeError('Expected: {}'.format(str(self.expected_types)))
self.is_set = True
super(Typed, self).__set__(instance, value)
Wrong, because when doing the following, ImmutableTyped is 'global' in the sense that it's a singleton throughout all instances of the class. When t2 is instantiated, is_set is already True from the previous object.
class T(object):
v = ImmutableTyped(int)
def __init__(self, v):
self.v = v
t1 = T()
t2 = T() # fail when instantiating
Attempt #2: Thought instance in __set__ refers to the class containing the attribute so tried to check if instance.__dict__[self.name] is still a Typed. That is also wrong.
Idea #3: Make Typed be used more similar to #property by accepting a 'fget' method returning the __dict__ of T instances. This would require the definition of a function in T similar to:
#Typed
def v(self):
return self.__dict__
which seems wrong.
How to implement immutability AND type checking as a descriptor?
Now this is my approach to the problem:
class ImmutableTyped:
def __set_name__(self, owner, name):
self.name = name
def __init__(self, *, immutable=False, types=None)
self.immutable == immutable is True
self.types = types if types else []
def __get__(self, instance, owner):
return instance.__dict__[self.name]
def __set__(self, instance, value):
if self.immutable is True:
raise TypeError('read-only attribute')
elif not any(isinstance(value, cls)
for cls in self.types):
raise TypeError('invalid argument type')
else:
instance.__dict__[self.name] = value
Side note: __set_name__ can be used to allow you to not specify the attribute name in initialisation. This means you can just do:
class Foo:
bar = ImmutableTyped()
and the instance of ImmutableTyped will automatically have the name attribute bar since I typed for that to occur in the __set_name__ method.
Could not succeed in making such a descriptor. Perhaps it's also unnecessarily complicated. The following method + property use suffices.
# this also allows None to go through
def check_type(data, expected_types):
if data is not None and not isinstance(data, expected_types):
raise TypeError('Expected: {}'.format(str(expected_types)))
return data
class A():
def __init__(self, value=None):
self._value = check_type(value, (str, bytes))
#property
def value(self):
return self._value
foo = A()
print(foo.value) # None
foo.value = 'bla' # AttributeError
bar = A('goosfraba')
print(bar.value) # goosfraba
bar.value = 'bla' # AttributeError
class ImmutableTyped(object):
def __set_name__(self, owner, name):
self.name = name
def __init__(self, *, types=None):
self.types = tuple(types or [])
self.instances = {}
return None
def __get__(self, instance, owner):
return instance.__dict__[self.name]
def __set__(self, instance, value):
is_set = self.instances.setdefault(id(instance), False)
if is_set:
raise AttributeError("read-only attribute '%s'" % (self.name))
if self.types:
if not isinstance(value, self.types):
raise TypeError("invalid argument type '%s' for '%s'" % (type(value), self.name))
self.instances[id(instance)] = True
instance.__dict__[self.name] = value
return None
Examples:
class Something(object):
prop1 = ImmutableTyped(types=[int])
something = Something()
something.prop1 = "1"
Will give:
TypeError: invalid argument type '<class 'str'>' for 'prop1'
And:
something = Something()
something.prop1 = 1
something.prop1 = 2
Will give:
TypeError: read-only attribute 'prop1'
I can't set the right properties of an instance when setting their attributes via setattr in a factory method.
Given the following code where data is a simple dict containing e.g. { "age": "64", ...}
def factory(data):
obj = MyClass()
for k, v in data.items():
setattr(obj, k, v)
return obj
class MyClass(object):
def __init__(self):
self._age = None
# more...
#property
def age(self):
return self._age
#age.setter
def age(self, value):
some_validation(value)
self._age = value
def __setattr__(self, name, value):
object.__setattr__(self, name, value)
def __getitem__(self, item):
return self.__dict__.get(item, None)
def __getattr__(self, item):
self.__dict__[item] = None
return None
def __str__(self):
return json.dumps(self, default=lambda o: o.__dict__)
c = factory(data)
print(c)
I always get the following output when printing the created object:
{"_age": "64", ...}
But I need to have
{"age": "64", ...}
Why does the setattr method assign the leading underscore?
Some of the things you are trying to achieve get mixed up, like wanting to print __dict__ for a readable representation, but using private attributes for properties. Let's start from scratch and see how we can implement your class correctly.
You are trying to implement a class which attributes can be accessed both as keys and attributes. That is fine and can be accomplished in a more concise way.
class MyClass:
...
def __getitem__(self, item):
return self.__getattribute__(item)
def __setitem__(self, key, value):
return self.__setattr__(key, value)
You also want None to be returned when an attribute does not exist. This is covered by __getattr__ which is called exactly when an attribute does not exist.
def __getattr__(self, _):
return None
Then you want to add some validation to some attributes with property. It is indeed the correct way to proceed.
#property
def age(self):
return self._age
#age.setter
def age(self, value):
# some validation here
self._age = value
And finally you want to be able to have a nice string representation of your instance. We have to be careful for that since we had to add some private attributes that we do not want to print.
What we are going to do is implement a method keys to allow casting to dict. This method will only return keys for attributes which are not private nor methods.
def keys(self):
return [k for k in dir(self) if not k.startswith('_') and not callable(self[k])]
def __str__(self):
return json.dumps(dict(self))
This does the right thing.
obj = MyClass()
obj.age = 3
print(obj)
# prints: {"age": 3}
Disclaimer:
This article is more a recipe than a question, but I found the subject quite interesting, with almost no references in the Web.
If there is any better place on StackOverflow to publish this kind of articles, please let me know.
Subject:
How can I force Python to invoke different function depending on the type of attribute access (using class or instance) - e.g. force Python to invoke different method for MyClass.my_method() and MyClass().my_method()?
Usecase:
Let's say, we have custom Enum implementation (based on Python36 Enum, but with some customization). As a user of this Enum, we want to create a CustomEnum, inherit not just from Enum, but also from str: class MyEnum(str, Enum).We also want to add encoding and decoding feature. Our idea is to use MyEnum.encode to encode any object, that includes our enum members, but leave the original str.encode in power for instances of our enum class.
In short: MyEnum.encode invoke our custom encoding function, and have perfectly sens, from this point of view. MyEnum() is a string, so MyEnum().encode should invoke encode function inherited from str class.
Solution:
Write a descriptor, which will work as a switch.
Full answer in my first post.
Solution:
As far as I know, descriptors are the only objects, that can distinguish, if they are invoke for class or instance, because of the __get__ function signature: __get__(self, instance, instance_type). This property allows us to build a switch on top of it.
class boundmethod(object):
def __init__(self, cls_method=None, instance_method=None, doc=None):
self._cls_method = cls_method
self._instance_method = instance_method
if cls_method:
self._method_name = cls_method.__name__
elif instance_method:
self._method_name = instance_method.__name__
if doc is None and cls_method is not None:
doc = cls_method.__doc__
self.__doc__ = doc
self._method = None
self._object = None
def _find_method(self, instance, instance_type, method_name):
for base in instance_type.mro()[1:]:
method = getattr(base, method_name, None)
if _is_descriptor(method):
method = method.__get__(instance, base)
if method and method is not self:
try:
return method.__func__
except AttributeError:
return method
def __get__(self, instance, instance_type):
if instance is None:
self._method = self._cls_method or self._find_method(instance, instance_type, self._method_name)
self._object = instance_type
else:
self._method = self._instance_method or self._find_method(instance, instance_type, self._method_name)
self._object = instance
return self
#staticmethod
def cls_method(obj=None):
def constructor(cls_method):
if obj is None:
return boundmethod(cls_method, None, cls_method.__doc__)
else:
return type(obj)(cls_method, obj._instance_method, obj.__doc__)
if isinstance(obj, FunctionType):
return boundmethod(obj, None, obj.__doc__)
else:
return constructor
#staticmethod
def instance_method(obj=None):
def constructor(instance_method):
if obj is None:
return boundmethod(None, instance_method, instance_method.__doc__)
else:
return type(obj)(obj._cls_method, instance_method, obj.__doc__)
if isinstance(obj, FunctionType):
return boundmethod(None, obj, obj.__doc__)
else:
return constructor
def __call__(self, *args, **kwargs):
if self._method:
try:
return self._method(self._object, *args, **kwargs)
except TypeError:
return self._method(*args, **kwargs)
return None
Example:
>>> class Walkmen(object):
... #boundmethod.cls_method
... def start(self):
... return 'Walkmen start class bound method'
... #boundmethod.instance_method(start)
... def start(self):
... return 'Walkmen start instance bound method'
>>> print Walkmen.start()
Walkmen start class bound method
>>> print Walkmen().start()
Walkmen start instance bound method
I hope it will help some o you guys.
Best.
I actually just asked this question (Python descriptors and inheritance I hadn't seen this question). My solution uses descriptors and a metaclass for inheritance.
from my answer:
class dynamicmethod:
'''
Descriptor to allow dynamic dispatch on calls to class.Method vs obj.Method
fragile when used with inheritence, to inherit and then overwrite or extend
a dynamicmethod class must have dynamicmethod_meta as its metaclass
'''
def __init__(self, f=None, m=None):
self.f = f
self.m = m
def __get__(self, obj, objtype=None):
if obj is not None and self.f is not None:
return types.MethodType(self.f, obj)
elif objtype is not None and self.m is not None:
return types.MethodType(self.m, objtype)
else:
raise AttributeError('No associated method')
def method(self, f):
return type(self)(f, self.m)
def classmethod(self, m):
return type(self)(self.f, m)
def make_dynamicmethod_meta(meta):
class _dynamicmethod_meta(meta):
def __prepare__(name, bases, **kwargs):
d = meta.__prepare__(name, bases, **kwargs)
for base in bases:
for k,v in base.__dict__.items():
if isinstance(v, dynamicmethod):
if k in d:
raise ValueError('Multiple base classes define the same dynamicmethod')
d[k] = v
return d
return _dynamicmethod_meta
dynamicmethod_meta=make_dynamicmethod_meta(type)
class A(metaclass=dynamicmethod_meta):
#dynamicmethod
def a(self):
print('Called from obj {} defined in A'.format(self))
#a.classmethod
def a(cls)
print('Called from class {} defined in A'.format(cls))
class B(A):
#a.method
def a(self):
print('Called from obj {} defined in B'.format(self))
A.a()
A().a()
B.a()
B().a()
results in:
Called from class <class 'A'> defined in A
Called from obj <A object at ...> defined in A
Called from class <class 'B'> defined in A
Called from obj <B object at ...> defined in B
In the example enumeration code given in this question, reproduced below, why does TOKEN contain the implementations of __contains__ and __repr__ from the metaclass EnumerationType?
from ctypes import *
class EnumerationType(type(c_uint)):
def __new__(metacls, name, bases, dict):
if not "_members_" in dict:
_members_ = {}
for key,value in dict.items():
if not key.startswith("_"):
_members_[key] = value
dict["_members_"] = _members_
cls = type(c_uint).__new__(metacls, name, bases, dict)
for key,value in cls._members_.items():
globals()[key] = value
return cls
def __contains__(self, value):
return value in self._members_.values()
def __repr__(self):
return "<Enumeration %s>" % self.__name__
class Enumeration(c_uint):
__metaclass__ = EnumerationType
_members_ = {}
def __init__(self, value):
for k,v in self._members_.items():
if v == value:
self.name = k
break
else:
raise ValueError("No enumeration member with value %r" % value)
c_uint.__init__(self, value)
#classmethod
def from_param(cls, param):
if isinstance(param, Enumeration):
if param.__class__ != cls:
raise ValueError("Cannot mix enumeration members")
else:
return param
else:
return cls(param)
def __repr__(self):
return "<member %s=%d of %r>" % (self.name, self.value, self.__class__)
class TOKEN(Enumeration):
_members_ = {'T_UNDEF':0, 'T_NAME':1, 'T_NUMBER':2, 'T_STRING':3, 'T_OPERATOR':4, 'T_VARIABLE':5, 'T_FUNCTION':6}
I would expect to have exceptions thrown by the following code to the effect that __contains__ is not implemented, instead however, I receive True False.
print 2 in TOKEN, 7 in TOKEN
Both Enumeration and TOKEN are instances of EnumerationType:
>>> isinstance(Enumeration, EnumerationType)
True
>>> isinstance(TOKEN, EnumerationType)
True
And special methods for instances of new style classes are looked up in class, e.g. repr(TOKEN) is equivalent to type(TOKEN).__repr__(TOKEN), which is EnumerationType.__repr__(TOKEN).