So I have a situation where one module writes some code that processes records, so lets say it does this:
from datetime import datetime
def is_record_today(rec: Record) -> bool:
return (rec.date == str(datetime.now().date()))
def is_record_valid(rec: Record) -> bool:
return record.is_valid()
so at this time I need to define:
import abc
class Record(abc.ABC):
#abc.abstractproperty
def date(self) -> str:
print('I am abstract so should never be called')
#abc.abstractmethod
def is_valid(self) -> bool:
print('I am abstract so should never be called')
now when I am processing a record in another module I want to inherit from this abstract class so I raise an error if I don't have date and is_valid. However, I also want to inherit from another class to get the default properties, lets say I want my record to be manipulatable like a dictionary. Then:
class ValidRecord(Record, dict):
#property
def date(self) -> str:
return self['date']
def is_valid(self) -> bool:
return self['valid']
class InvalidRecord(Record, dict):
#property
def date(self) -> str:
return self['date']
we would expect that ValidRecord should create without issue and InvalidRecord should throw TypeError, however both seem fine, and I can even call the missing abstractmethod from the Record class which as far as I understand abstract methods should not be possible:
data = {
'date': '2021-05-01',
'valid': False
}
InValidRecord(data).is_valid()
>>> "I am abstract so should never be called"
If I take away the dictionary inheritance I get the expected error so, what is going on here and how can I get the desired behavior of inheriting from one class but requiring additional methods be added?
Related
Imagine I have the following class structure with three class families: Parent, Child1, and Child2 each with Abstract, Explicit and Implicit flavors.
from __future__ import annotations
import abc
###################################
# Parent classes
###################################
class AbstractParent(abc.ABC):
#property
#abc.abstractmethod
def type_explicit(self) -> type[AbstractExplicitParent]:
pass
class AbstractExplicitParent(AbstractParent):
pass
class AbstractImplicitParent(AbstractParent):
pass
###################################
# Child 1 classes
###################################
class AbstractChild1(AbstractParent):
#property
def type_explicit(self) -> type[ExplicitChild1]:
return ExplicitChild1
class ExplicitChild1(AbstractExplicitParent, AbstractChild1):
pass
class ImplicitChild1(AbstractImplicitParent, AbstractChild1):
pass
###################################
# Child 2 classes
###################################
class AbstractChild2(AbstractParent):
#property
def type_explicit(self) -> type[ExplicitChild2]:
return ExplicitChild2
class ExplicitChild2(AbstractExplicitParent, AbstractChild2):
pass
class ImplicitChild2(AbstractImplicitParent, AbstractChild1):
pass
The details of these classes aren't important except for the fact that I've defined an abstract property called type_explicit in AbstractParent with concrete implementations in AbstractChild1 and AbstractChild2. The obvious purpose of this property is to return the explicit flavor of the current class, and you can see that AbstractChild1.type_explicit returns ExplicitChild1 and similarly for AbstractChild2.type_explicit.
Now imagine I want to define a function that can accept any subclass of AbstractParent but always returns the explicit flavor of that subclass. Obviously I can write
def some_function(a: AbstractParent) -> AbstractExplicitParent:
...
but that doesn't capture all the type information, for example: (ImplicitChild1,) -> ExplicitChild1.
Is there a way to dynamically compute the return type, say by defining a (meta)class called Explicit that can evaluate the type_explicit property, such that I can write
T = TypeVar("T", bound=AbstractParent)
def some_function(a: T) -> Explicit[T]:
...
even if I need to change how type_explicit works, say by changing it to a class variable/method?
The only reason I think this could work is because of things like typing.Type which are sort of dynamic since I can write things like
def get_type(a: T) -> Type[T]:
return type(a)
I want a simple way to implement new filters in a module. They would eventually be automatically recognized by the library at import.
For example, if I want the list of all filters I do:
>>> FilterFactory.available_filters
{
'upper': __main__.FilterUpper,
'lower': __main__.FilterLower,
'trim': __main__.FilterTrim
}
My first approach was to use a classmethod and a LRU Cache:
class FilterFactory:
#classmethod
#lru_cache()
def available_filters(cls):
fmap = {}
for _, member in inspect.getmembers(sys.modules[__name__]):
if not inspect.isclass(member) or not hasattr(member, 'name'):
continue
if member.name() == 'base':
continue
fmap[member.name()] = member
return fmap
Then I realized that it is better to build the factory when the module is loaded using metaclasses:
from abc import abstractmethod
class FilterFactory:
available_filters = {}
#classmethod
def register(cls, filter_: type):
# if not issubclass(filter_, Filter):
# raise InvalidFilterError(f'Invalid filter: {filter_}')
cls.available_filters[filter_.name] = filter_
setattr(cls, filter_.name, filter_)
def __new__(cls, name, *args, **kwargs):
if name not in cls.available_filters:
raise ValueError(f'Unknown filter: {name}')
return cls.available_filters[name](*args, **kwargs)
class MetaFilter(type):
def __new__(cls, name, bases, attrs):
new_class = super().__new__(cls, name, bases, attrs)
if not name.startswith('Filter') and name != 'BaseFilter':
raise ValueError('Filter class names must start with "Filter"')
new_class.name = name.split('Filter', maxsplit=1)[1].lower()
if name != 'BaseFilter':
FilterFactory.register(new_class)
return new_class
class BaseFilter(metaclass=MetaFilter):
""" Base class for filters. """
#abstractmethod
def filter(self, value: str) -> str:
raise NotImplementedError("Filter.filter() must be implemented")
def __init__(self, *args, **kwargs):
...
def __repr__(self):
return f'{self.__class__.__name__}'
def __call__(self, value: str) -> str:
return self.filter(value)
class FilterUpper(BaseFilter):
def filter(self, value: str) -> str:
return value.upper()
class FilterRegex(BaseFilter):
def __init__(self, pattern: str, replace: str):
self.pattern = re.compile(pattern)
self.replace = replace
def filter(self, value: str) -> str:
return self.pattern.sub(value, self.replace)
This looks neat, but it has some flaws:
I cannot ensure the filter passed to register is indeed a subclass of BaseFilter because this base class isn't yet declared. Unlike C++ I cannot do forward declarations in Python.
I must specifically prevent the abstract class BaseFilter to be added to the available_filters.
This pattern looks a bit odd.
The goal is to be able to use FilterFactory.available_filters to build a JSON schema validator that only accepts available filters. And use the factory to create then apply filters multiple times during the execution of the program. The validation may be done with voluptuous by adding some extra in the metaclass:
class MetaFilter(type):
def __new__(cls, name, bases, attrs):
...
new_class.__params__, new_class.__types__ =
cls.extract_parameters(new_class)
return new_class
#classmethod
def extract_parameters(cls, new_class):
""" Extract parameters from the class.
Ensure that all the parameters are annotated."""
params = dict(inspect.signature(new_class.__init__).parameters)
for key in ['self', 'args', 'kwargs']:
if key in params:
del params[key]
for param, value in params.items():
if value.annotation is inspect.Parameter.empty:
raise ValueError(
f'Filter {new_class.name} has an untyped parameter: {param}'
)
return (params.keys(), [p.annotation for p in params.values()])
Then I can create a validation schema with:
filters = {}
for filter_name, filter_class in FilterFactory.available_filters.items():
filters[Optional(filter_name)] = All(
ExactSequence(filter_class.__types__),
lambda args: FilterFactory(filter_name, *args)
)
schema = Schema({'filter': filters})
s = schema({
'filter': {
'regex': ['foo', 'bar']
}
})
assert(s['filter']['regex'].filter('foo') == 'bar')
If the filter is missing from the implementation, the validation fails. Adding a new filter to the application is as simple as adding this filter in the filters.py module.
Is this implementation Zen and Pythonic? What better option can I use?
TL;DR:
The idea is good - I don't see the problem of "can't forward reference classes" as a real one,a s a filter class will have to import BaseFilter anyway, even if it is in a different file, and therefore, it has to be made available early, or the program won't even run. (that is: you won't get a class declared as inheriting from BaseFilter that dos not, in fact, does so).
That said, since Python 3.6 there is a new feature in the language that does away with the need for a metaclass in this case (and as a bonus, it even simplifies the fact that BaseFilter itself is not a filter): the __init_subclass__ method.
It should be written as a plain method on a base-class - it will always be a class method, even without being decorated with #classmethod, and it will be called for each new subclass, with the subclass as first argument: you can write all your registering logic in that method. (ANd it is not called for the base class, where it should be declared, itself).
init subclass documentation
I am creating a dynamic class from an abstract base class. Here is a simplified example.
from abc import ABC
from typing import List, Type
class IParentClass(ABC):
_children: List['IParentClass'] = []
#property
def children(self) -> List['IParentClass']:
return self._children
def add_child(self) -> None:
self._children.append(self._create_child()())
#classmethod
def _create_child(cls: Type['IParentClass']) -> Type['IParentClass']:
class DynamicChild(cls):
pass
return DynamicChild
class RealChild(IParentClass):
pass
rc = RealChild()
rc.add_child()
rc.children[0].add_child()
rc.children[0].children[0]
The code works, but Mypy gives me two errors (Variable "cls" is not valid as a type, and Invalid base class "cls") on cls in _create_child.
I could ignore the error, but I was wondering if there is a better way to manage this.
I'm trying to define a couple of dataclasses and an abstract class that manipulates those classes. Eventually, the my_class_handler types could be dealing with say: json, xml or sqlite files as concrete instance types.
Can someone please explain to me what this message means?
<bound method my_class_handler.class_name of <__main__.my_class_handler object at 0x000001A55FB96580>>
Here's the source code that generates the error for me.
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import List
#dataclass
class column:
name: str
heading: str
#dataclass
class my_class:
class_name: str
class_description: str
columns: List[column]
class iclass_handler(ABC):
#abstractmethod
def class_name(self) -> str:
pass
#abstractmethod
def class_name(self, value: str):
pass
class my_class_handler(iclass_handler):
obj: my_class
def __init__(self):
self.obj = my_class("test-class", "", None)
def class_name(self) -> str:
return self.obj.class_names
def class_name(self, value: str):
if (value != self.obj.class_name):
self.obj.class_name = value
if __name__ == '__main__':
handler = my_class_handler()
print(handler.class_name)
If this is not the proper way of doing this, please point me in the direction where I might learn the proper way.
Thanks for your time,
Python does not allow overloading like Java, so remove methods that overlap.
#khelwood pointed out the answer to the original question. Thanks
As for the #property approach, I tried that and was having nothing but problems and couldn't find any useful examples of inherited properties so I just rewrote the function to take an additional parameter:
# I'm working from memory here but I believe this is the jist...
def class_name(self, new_value: str = None) -> str:
if (new_value is None)
return self.obj.class_name
if (isinstance(new_value, str)):
if (new_value != self.obj.class_name):
self.obj.class_name = new_value
return None
Anyhow, I have since refactored and have completely removed the whole class_name() method as a result of a redesign that dropped the whole concept of data-handlers.
Thanks again for the comments.
Coming from a C# background and knowing its generic type approaches I'm now trying to implement something similar in Python. I need to serialize and de-serialize classes in a special string format, so I created the following two base classes, the first for single entity serialization and the second one for list serialization of that entity type.
from typing import Any, TypeVar, List, cast, Type, Generic, NewType
import re
T = TypeVar('T')
class Serializable(Generic[T]):
def to_str(self) -> str:
raise NotImplementedError
#classmethod
def from_str(cls, str: str):
raise NotImplementedError
class SerializableList(List[Serializable[T]]):
def __init__(self):
self.separator: str = "\n"
#classmethod
def from_str(cls, str: str):
list = cls()
for match in re.finditer(list.separator, str):
list.append(T().from_str(match)) # <-- PROBLEM: HOW TO INIT A GENERIC ENTITY ???
# list.append(Serializable[T].from_str(match)) <-- Uses base class (NotImplemented) instead of derived class
return list
def to_str(self) -> str:
str = ""
for e in self:
str = str + f"{e.to_str()}{self.separator}"
return str
Then I can derive from those classes and have to implement to_str and from_str. Please see the marker <-- PROBLEM". I have no idea how I can init a new entity of the currently used type for the list. How do we do this in the Python way?
As #user2357112supportsMonica says in the comments, typing.Generic is pretty much only there for static analysis, and has essentially no effect at runtime under nearly all circumstances. From the look of your code, it looks like what you're doing might be better suited to Abstract Base Classes (documentation here, tutorial here), which can be easily combined with Generic.
A class that has ABCMeta as its metaclass is marked as an Abstract Base Class (ABC). A subclass of an ABC cannot be instantiated unless all methods in the ABC marked with the #abstractmethod decorator have been overridden. In my suggested code below, I've explicitly added the ABCMeta metaclass to your Serializable class, and implicitly added it to your SerializableList class by having it inherit from collections.UserList instead of typing.List. (collections.UserList already has ABCMeta as its metaclass.)
Using ABCs, you could define some interfaces like this (you won't be able to instantiate these because of the abstract methods):
### ABSTRACT INTERFACES ###
from abc import ABCMeta, abstractmethod
from typing import Any, TypeVar, Type, Generic
from collections import UserList
import re
T = TypeVar('T')
class AbstractSerializable(metaclass=ABCMeta):
#abstractmethod
def to_str(self) -> str: ...
#classmethod
#abstractmethod
def from_str(cls: Type[T], string: str) -> T: ...
S = TypeVar('S', bound=AbstractSerializable)
class AbstractSerializableList(UserList[S]):
separator = '\n'
#classmethod
#property
#abstractmethod
def element_cls(cls) -> Type[S]: ...
#classmethod
def from_str(cls, string: str):
new_list = cls()
for match in re.finditer(cls.separator, string):
new_list.append(cls.element_cls.from_str(match))
return new_list
def to_str(self) -> str:
return self.separator.join(e.to_str() for e in self)
You could then provide some concrete implementations like this:
class ConcreteSerializable(AbstractSerializable):
def to_str(self) -> str:
# put your actual implementation here
#classmethod
def from_str(cls: Type[T], string: str) -> T:
# put your actual implementation here
class ConcreteSerializableList(AbstractSerializableList[ConcreteSerializable]:
# this overrides the abstract classmethod-property in the base class
element_cls = ConcreteSerializable
(By the way — I changed several of your variable names — str, list, etc — as they were shadowing builtin types and/or functions. This can often lead to annoying bugs, and even if it doesn't, is quite confusing for other people reading your code! I also cleaned up your to_str method, which can be simplified to a one-liner, and moved your separator variable to be a class variable, since it appears to be the same for all class instances and does not appear to ever be altered.)
For now I found a dirty solution - this is to add a Type (constructor) parameter of the list entries like so:
class SerializableList(List[Serializable[T]]):
# This one
# |
# v
def __init__(self, separator: str = "\n", entity_class: Type = None):
self.separator = separator
self.entity_class = entity_class
#classmethod
def from_str(cls, str: str):
list = cls()
for match in re.finditer(list.separator, str):
list.append(list.entity_class.from_str(match))
return list
I wonder if there is a cleaner way to get the correct [T] type constructor from List[T] since it is already provided there?