Develop Reference

python3 undefined __ne__ returns same value as defined __eq__

I have a class MyClass where __eq__ is defined as follows:
def __eq__(self, other):
try:
other = MyClass(other)
except Exception as e:
raise ValueError(f"MyClass.__eq__ failed: {other} doesn't appear to be a valid MyClass type. \n", e )
prefix_i1 = re.findall("[ab]" , self )
prefix_i2 = re.findall("[ab]" , other )
if len( set( prefix_i1 + prefix_i2 ) ) > 1:
return False # ie: "a07" != "b07" BUT "k07" == "b07"
else:
return self.num_string == other.num_string # eg: "i3" == "i03"
where
self.num_string = "".join(filter(str.isdigit, self )).zfill(2)
Unexpectedly, I get this result:
> MyClass('b05') == MyClass('i05')
True
> MyClass('b05') != MyClass('i05')
True
I thought __ne__ was by default not __eq__ and defining it explicitly was discouraged since Python 3.
What am I doing wrong?
Here is a minimal reproducible example:
import re
class MyClass(str):
def __new__(cls, *args, **kwargs):
if args[0] is None:
return str.__new__(cls, '' )
if not ( isinstance(args[0], int) or
( isinstance(args[0], str) and re.match("[iab]?[0-9]{1,2}", args[0].lower()) ) ):
raise ValueError("MyClass failed because the provided input '{}' is not valid... .".format(args[0]))
lower_str_input = str(args[0]).lower()
return str.__new__(cls, lower_str_input )
def __init__(self, input_string=''):
self.num_string = "".join(filter(str.isdigit, self )).zfill(2)
def __eq__(self, other):
try:
other = MyClass(other)
except Exception as e:
raise ValueError("MyClass.__eq__ failed" )
prefix_i1 = re.findall("[ab]" , self )
prefix_i2 = re.findall("[ab]" , other )
if len( set( prefix_i1 + prefix_i2 ) ) > 1:
return False # ie: "a07" != "b07" BUT "k07" == "b07"
else:
return self.num_string == other.num_string # eg: "i3" == "i03"
MyClass('b05') == MyClass('i05')
MyClass('b05') != MyClass('i05')

I thought __ne__ was by default not __eq__ and defining it explicitly was discouraged since Python 3.
For your own classes, yes. They will derive from object by default, which implements that logic.
However, built-ins (and thus everything derived from them) can be a bit strange.
Here is a simpler example:
... class x(str):
... def __eq__(self, other):
... print('eq called')
... return str.__eq__(self, other)
...
>>> x('foo') != 'foo'
False
The answer is correct, but __eq__ is not called (thus there is no message printed). This answer comes from str.__ne__, which (apparently) is separately implemented (not in terms of __eq__).

Recursive class in python and a bug in thrift

Thrift 0.9.2 now allows for recursive structs that can contain instances of themselves. For example this thrift definition:
namespace py foo
struct Foo {
1: optional string member
2: optional Foo fooData
}
Produces the following code:
#
# Autogenerated by Thrift Compiler (0.9.2)
#
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
#
# options string: py
#
from thrift.Thrift import TType, TMessageType, TException, TApplicationException
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol, TProtocol
try:
from thrift.protocol import fastbinary
except:
fastbinary = None
class Foo:
"""
Attributes:
- member
- fooData
"""
thrift_spec = (
None, # 0
(1, TType.STRING, 'member', None, None, ), # 1
(2, TType.STRUCT, 'fooData', (Foo, Foo.thrift_spec), None, ), # 2
)
def __init__(self, member=None, fooData=None,):
self.member = member
self.fooData = fooData
def read(self, iprot):
if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
return
iprot.readStructBegin()
while True:
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 1:
if ftype == TType.STRING:
self.member = iprot.readString();
else:
iprot.skip(ftype)
elif fid == 2:
if ftype == TType.STRUCT:
self.fooData = Foo()
self.fooData.read(iprot)
else:
iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
iprot.readStructEnd()
def write(self, oprot):
if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
return
oprot.writeStructBegin('Foo')
if self.member is not None:
oprot.writeFieldBegin('member', TType.STRING, 1)
oprot.writeString(self.member)
oprot.writeFieldEnd()
if self.fooData is not None:
oprot.writeFieldBegin('fooData', TType.STRUCT, 2)
self.fooData.write(oprot)
oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
def validate(self):
return
def __hash__(self):
value = 17
value = (value * 31) ^ hash(self.member)
value = (value * 31) ^ hash(self.fooData)
return value
def __repr__(self):
L = ['%s=%r' % (key, value)
for key, value in self.__dict__.iteritems()]
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
def __eq__(self, other):
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
def __ne__(self, other):
return not (self == other)
The problem with this code is in the class variable thrift_spec:
thrift_spec = (
None, # 0
(1, TType.STRING, 'member', None, None, ), # 1
(2, TType.STRUCT, 'fooData', (Foo, Foo.thrift_spec), None, ), # 2
)
There are two problems here:
The class def tries to reference Foo before it is fully defined (and will produce NameError: name 'Foo' is not defined) when instantiated.
thrift_spec tries to contain a reference to itself which is a bizarre thing to do in python (self-referential tuple?).
This tuple is used by a package in thrift called fastbinary, a C-module that serializes and deserializes thrift objects natively. The tuple is used to pass along type information to the C-module.
The question I have is how best to solve these two problems. For #1 I can move the instantiation of thrift_spec after the class definition or could move it into an #property. For #2 I'm really not sure what to do. Is there some way to get a reference to the tuple object and pass that to the C-module?
Note this is an open bug in thrift that I was hoping to help solve:
https://issues.apache.org/jira/browse/THRIFT-2642

Is there a Python equivalent for Scala's Option or Either?

I really enjoy using the Option and Either monads in Scala. Are there any equivalent for these things in Python? If there aren't, then what is the pythonic way of handling errors or "absence of value" without throwing exceptions?

The pythonic way for a function to say "I am not defined at this point" is to raise an exception.
>>> int("blarg")
Traceback (most recent call last):
...
ValueError: invalid literal for int() with base 10: 'blarg'
>>> dict(foo=5)['bar']
Traceback (most recent call last):
...
KeyError: 'bar'
>>> 1 / 0
Traceback (most recent call last):
...
ZeroDivisionError: integer division or modulo by zero
This is, in part, because there's no (generally useful) static type checker for python. A Python function cannot syntactically state, at compile time, that it has a particular codomain; there's no way to force callers to match all of the cases in the function's return type.
If you prefer, you can write (unpythonically) a Maybe wrapper:
class Maybe(object):
def get_or_else(self, default):
return self.value if isinstance(self, Just) else default
class Just(Maybe):
def __init__(self, value):
self.value = value
class Nothing(Maybe):
pass
But I would not do this, unless you're trying to port something from Scala to Python without changing much.

You can play with typing package (Python 3.6.9). Using following makes type checker happy
from typing import Optional, Union
def parse_int(s: str) -> Optional[int]:
try:
return int(s)
except:
return None
print('-- optional --')
print(parse_int('123'))
print(parse_int('a'))
def parse_int2(s: str) -> Union[str, int]:
try:
return int(s)
except Exception as e:
return f'Error during parsing "{s}": {e}'
print('-- either --')
print(parse_int2('123'))
print(parse_int2('a'))
Result
-- optional --
123
None
-- either --
123
Error during parsing "a": invalid literal for int() with base 10: 'a'
If you want to add monadic behaviour to Either you can try this
from typing import TypeVar, Generic, Callable
A = TypeVar('A')
B = TypeVar('B')
C = TypeVar('C')
Either = NewType('Either', Union['Left[A]', 'Right[C]'])
class Left(Generic[A]):
def __init__(self, value: A):
self.__value = value
def get(self) -> A:
raise Exception('it is left')
def get_left(self) -> A:
return self.__value
def flat_map(self, f: Callable[[B], Either]) -> Either:
return self
def map(self, f: Callable[[B], C]) -> Either:
return self
def __str__(self):
return f'Left({self.__value})'
and right type
class Right(Generic[B]):
def __init__(self, value: B):
self.__value = value
def flat_map(self, f: Callable[[B], Either]) -> Either:
return f(self.__value)
def map(self, f: Callable[[B], C]) -> Either:
return Right(f(self.__value))
def __str__(self):
return f'Right({self.__value})'
def parse_int(s: str) -> Union[Left[str], Right[int]]:
try:
return Right(int(s))
except Exception as e:
return Left(f'Error during parsing {s}: {e}')
def divide(x: int) -> Union[Left[str], Right[int]]:
return Right(4 / x) if x != 0 else Left('zero !!!')
print(parse_int('1').map(lambda x: x * 2))
print(parse_int('a').map(lambda x: x * 2))
print(parse_int('2').flat_map(divide))
print(parse_int('0').flat_map(divide))
Result
Right(2)
Left(Error during parsing a: invalid literal for int() with base 10: 'a')
Right(2.0)
Left(zero !!!)

mypy adds type definitions and type checking (not at runtime) over regular Python. They have an Optional: https://docs.python.org/3/library/typing.html#typing.Optional. More here https://www.python.org/dev/peps/pep-0484/#rationale-and-goals. Intellij has plugin support which makes it all very professional and smooth.

In python, for an absence of value, the variable is None, so you can do it this way.
vars = None
vars = myfunction()
if vars is None:
print 'No value!'
else:
print 'Value!'
or even just check if a value is present like this
if vars is not None:
print vars

I realize this is pretty late to the party but I came to this page on top of google before deciding to implement it so maybe I can help others googling with this. I implemented it, you can get it from pypi as pyther-maybe, it implements both Either and Maybe with Maybe as a special subclass of Either. This example should explain how it works:
import sys
from pyther_maybe import *
def save_div ( x, y ):
if y == 0:
return nothing() # alias of Maybe()
else:
return value(x / y) # alias of Maybe(x / y)
float_test = save_div(1.0, 3.0)
assert isinstance(float_test, Maybe)
if float_test: #nothing tests as false:
float = float_test() # calling the container with no arguments returns its value
else:
sys.exit("something went wrong")
print float
# or if you want to encode a reason:
def save_div ( x, y ):
if y == 0:
return left("You can't divide by zero, silly") # alias of Either(left=...)
else:
return right(x / y) # alis of Either(...)
float_test = save_div(4.2, 0.0)
assert isinstance(float_test, Either)
def fake_exit ( string ):
print "We would have exited with:"
print string
return "Whatever value"
if float_test:
# these two are te same
float = float_test()
float = float_test.right()
else:
fake_exit(float_test.left())
# or in a shorter and more pleasant format
# does the same as above
float = float_test.extract(fake_exit)
print float # prints "Whatever value"
# Also, these containers are mutable:
box = nothing()
try:
print box() # raises exception
except RightEitherException:
print "We caught an exception"
def change_box():
box(4)
change_box()
print box() # 4
It has more features than that, some of which are pretty useless in practise (it's also an iterator for instance and has subscript notation like pyther_maybe.either(x)[pyther_maybe.Right] == x.

Try This:
from monad import Monad
class Either(Monad):
# pure :: a -> Either a
#staticmethod
def pure(value):
return Right(value)
# flat_map :: # Either a -> (a -> Either b) -> Either b
def flat_map(self, f):
if self.is_left:
return self
else:
return f(self.value)
class Left(Either):
def __init__(self, value):
self.value = value
self.is_left = True
class Right(Either):
def __init__(self, value):
self.value = value
self.is_left = False

A list that happens to always be of length zero or one fulfills some of the same goals as optional/maybe types. You won't get the benefits of static typing in Python, but you'll probably get a run-time error even on the happy path if you write code that tries to use the "maybe" without explicitly "unwrapping" it.

Natively, Python has a Literal Type Optional but it's not the same. Alternatively this is a representation of the Either data type for python 3.
https://gist.github.com/MatteoGuadrini/98e79a9ab2bd6ae5badc41df89cfe338

pip install either
The library was created in 2012, its Development status is: Stable, according to https://pypi.org/project/either/.

There is a package called Pymonad that implements option types as maybe and an either type as well as other functional ideas such as a wrapper for currying, monads, monoids, and others.
General Details
https://pypi.org/project/PyMonad/
Reference
https://jasondelaat.github.io/pymonad_docs/reference/pymonad.html

You can write your own Optional implement
from __future__ import annotations
from typing import *
T = TypeVar("T")
U = TypeVar("U")
class Option(Generic[T]):
_value: Optional[T]
#classmethod
def ofNullable(cls, value: Optional[T]) -> Option[T]:
option = cls()
option._value = value
return option
#classmethod
def of(cls, value: Optional[T]) -> Option[T]:
assert value is not None
return cls.ofNullable(value)
#classmethod
def empty(cls) -> Option[T]:
return cls.ofNullable(None)
def isPresent(self) -> bool:
return self._value is not None
def ifPresent(self, consumer: Callable[[T], Any]) -> None:
if self._value is not None:
consumer(self._value)
def get(self) -> T:
return self._value
def orElse(self, other: T) -> T:
return self._value if self._value is not None else other
def orElseGet(self, other: Callable[[], T]) -> T:
return self._value if self._value is not None else other()
def orElseThrow(self, exceptionSupplier: Callable[[], BaseException]):
if self._value is not None:
return self._value
else:
raise exceptionSupplier()
def filter(self, predicate: Callable[[T], bool]) -> Option[T]:
if predicate(self):
return self
return self.__class__.empty()
def map(self, mapper: Callable[[T], U]) -> Option[U]:
if self._value is not None:
return self.__class__.of(mapper(self._value))
return self.__class__.empty()
def flatMap(self, mapper: Callable[[T], Option[U]]) -> Option[U]:
if self._value is not None:
return mapper(self._value)
return self.__class__.empty()
def __str__(self) -> str:
return f"<Option:{self._value}>"
def __repr__(self) -> str:
return f"<Option:{repr(self._value)}>"
def __eq__(self, other: Union[T, Any]):
if isinstance(other, Option):
return self._value == other._value
return False
Or use a 3rd Optional library
IceSpringRealOptional
Real Optional type in python, not #Nullable annotation.
Official sites
Home: https://baijifeilong.github.io/2022/01/09/ice-spring-real-optional/index.html
Github: https://github.com/baijifeilong/IceSpringRealOptional
PyPI: https://pypi.org/project/IceSpringRealOptional
Features
All Java 8 style Optional API support
All Generic Type annotation support
Install
PyPI: pip install IceSpringRealOptional
Usage
from IceSpringRealOptional import Option
option = Option.ofNullable("CommonSense")
print("{}: isPresent={}".format(option, option.isPresent()))
print("{}: value={}".format(option, option.get()))
option.ifPresent(lambda x: print(f"{x} exist"))
print("{}'s length: {}".format(option, option.map(len)))
empty = Option.empty()
print(empty.orElse("{} is empty".format(empty)))
print(empty.orElseGet(lambda: "{} is empty again".format(empty)))
try:
Option.empty().orElseThrow(lambda: RuntimeError("Unlucky"))
except RuntimeError as e:
print("Runtime error caught: {}".format(e))
Example Output
<Option:CommonSense>: isPresent=True
<Option:CommonSense>: value=CommonSense
CommonSense exist
<Option:CommonSense>'s length: <Option:11>
<Option:None> is empty
<Option:None> is empty again
Runtime error caught: Unlucky

Ordering enum value in python

I would like to be able to arrange the ordering of Enum. Has somebody suggestions how this can be solved?
The following Enum meta class is using:
class EnumMeta(type):
def __new__(typ, name, bases, attrs):
cls_attrs = {}
cls_choices = []
for attr_name, value in attrs.items():
cls_attrs[attr_name] = attr_name.lower()
if not attr_name.startswith("__"):
cls_choices.append((attr_name.lower(), value))
def choices(cls):
return cls_choices
def values(cls, value=None):
if value is None:
return {choice[0]: unicode(choice[1]) for choice in cls.choices()}
elif isinstance(value, list):
return {choice[0]: unicode(choice[1]) for choice in cls.choices() if choice[0] in value}
else:
return unicode(dict(cls.choices()).get(value))
def keys(cls, nil=False):
items = [item[0] for item in cls.choices()]
if nil:
items.append('')
return items
def combined_length(cls):
return len(",".join(cls.values().keys()))
def max_length(cls):
return max(map(len, cls.values().keys()))
cls_attrs['choices'] = classmethod(choices)
cls_attrs['values'] = classmethod(values)
cls_attrs['keys'] = classmethod(keys)
cls_attrs['combined_length'] = classmethod(combined_length)
cls_attrs['max_length'] = classmethod(max_length)
return type(name, bases, cls_attrs)
An example of an Enum is as follow:
class SideHemType:
__ordering__ = ['double', 'single']
__metaclass__ = EnumMeta
Single = "Single side hem for opaque fabrics"
Double = "Double side hem for transparent fabrics"
class TestEnumOrdering:
print SideHemType.keys()
print SideHemType.values()
By printing the Enum SideHemType first Double is printed and then Single. But I would like first Single and then Double.

If you are using Python3.4 you can use the new enum.Enum type, which remembers the order the enum members are declared in.
If you are using an earlier Python, you should use the enum34 package available from PyPI, which supports Pythons back to 2.4.
The enum34 package, if used in Python3, also remembers the order of member declarations. If used in Python 2 it supports an extra _order_ attribute:
from enum import Enum
class SideHemType(Enum):
_order_ = 'Single Double' # only needed in Python 2
Single = "Single side hem for opaque fabrics"
Double = "Double side hem for transparent fabrics"
#classmethod
def combined_length(cls):
return len(",".join(mbr.name for mbr in cls))
#classmethod
def max_length(cls):
return max(map(len, (mbr.name for mbr in cls)))
print list(SideHemType) # [SideHemType.Single, SideHemType.Double]
print SideHemType.Double.value # "Double side hem for transparent fabrics"

Use IntEnum from the enum package and use the integer values to specify the order that you want:
class Shape(IntEnum):
CIRCLE = 1
SQUARE = 2
Shape.CIRCLE < Shape.SQUARE
Prints True.

How about this
class MyEnum(enum.Enum):
first_item = 'bla bla'
whatever = 'blubb'
another_one = 'blobb'
def __lt__(self, other: 'MyEnum'):
if self == other:
return False
# the following works because the order of elements in the definition is preserved
for elem in MyEnum:
if self == elem:
return True
elif other == elem:
return False
raise RuntimeError('Bug: we should never arrive here') # I just like being pedantic

Your Enum loses the ordering in 3 places. First the attributes on the class body are stored in a dictionary, then you copy the items into another dictionary. Finally your values() returns a 3rd dictionary. A dictionary does not save ordering, and it is impossible to get the ordering of the attributes within the class body.
With this system the easiest is to have a variable
__ordering__ = [ 'single', 'double' ]
And make the values() return a list of tuples (like dict.items()).
class EnumMeta(type):
def __new__(typ, name, bases, attrs):
cls_attrs = {}
cls_choices = {}
for attr_name, value in attrs.items():
cls_attrs[attr_name] = attr_name.lower()
if not attr_name.startswith("__"):
cls_choices[attr_name.lower()] = value
ordering = attrs.get('__ordering__')
if ordering == None:
ordering = sorted(cls_choices.keys())
def choices(cls):
return dict(cls_choices)
def values(cls, value=None):
if value is None:
return [ (k, cls_choices[k] ) for k in ordering ]
elif not isinstance(value, basestring):
return [ (k, cls_choices[k] ) for k in value ]
else:
return unicode(cls_choices.get(value))
def keys(cls, nil=False):
items = list(ordering)
if nil:
items.append('')
return items
def combined_length(cls):
return len(",".join(cls.values().keys()))
def max_length(cls):
return max(map(len, cls.values().keys()))
cls_attrs['choices'] = classmethod(choices)
cls_attrs['values'] = classmethod(values)
cls_attrs['keys'] = classmethod(keys)
cls_attrs['combined_length'] = classmethod(combined_length)
cls_attrs['max_length'] = classmethod(max_length)
return type(name, bases, cls_attrs)
class SideHemType:
__ordering__ = ['double', 'single']
__metaclass__ = EnumMeta
Single = "Single side hem for opaque fabrics"
Double = "Double side hem for transparent fabrics"
print SideHemType.keys()
print SideHemType.values()

Need to store bytestring using Django

I have a bytestring that i need to store.
Since Django does not support BlobFields I thought I'd create my own Base64Field,
that encodes and decodes to base64 upon interaction with the db.
So I overrode (?) the to_python and get_db_prep_save methods for that exact purpose.
Problem is that to_python gets called in various different scenarios, not just once and
there is no way to tell if the string is already decoded or not. If it is already decoded,
an error obviously occurs.
What solutions are there for my dilema?
Possible solutions that seem ugly to me: try except the decoding process, return value if decoding fails, using an instance variable only allow for 1 to_python (this seems even worse)

You can use some kind of PickledObjectField.
class PickledObjectField(models.Field):
__metaclass__ = models.SubfieldBase
marker_re = re.compile(r'^T\[(?P<type>\w+)\](?P<value>.*)$', re.DOTALL)
markable_types = dict((t.__name__, t) for t in (str, int, unicode))
def __init__(self, *args, **kwargs):
self.compress = kwargs.pop('compress', True)
self.protocol = kwargs.pop('protocol', 2)
kwargs.setdefault('null', True)
kwargs.setdefault('editable', False)
super(PickledObjectField, self).__init__(*args, **kwargs)
def generate_type_marked_value(self, value):
return PickledObject(u"T[%s]%s" % (type(value).__name__, value))
def read_marked_value(self, value):
m = self.marker_re.match(value)
if m:
marker = m.group('type')
value = m.group('value')
if marker in self.markable_types:
value = self.markable_types[marker](value)
return value
def get_default(self):
if self.has_default():
if callable(self.default):
return self.default()
return self.default
return super(PickledObjectField, self).get_default()
def to_python(self, value):
if value is not None:
try:
if value.startswith("T["):
value = self.read_marked_value(value)
else:
value = dbsafe_decode(value, self.compress)
except:
if isinstance(value, PickledObject):
raise
return value
def get_db_prep_value(self, value):
if value is not None and not isinstance(value, PickledObject):
if type(value).__name__ in self.markable_types and not (isinstance(value, basestring) and len(value
) > MAX_MARKABLE_STRING_LENGTH):
value = unicode(self.generate_type_marked_value(value))
else:
value = unicode(dbsafe_encode(value, self.compress))
return value
def value_to_string(self, obj):
value = self._get_val_from_obj(obj)
return self.get_db_prep_value(value)
def get_internal_type(self):
return 'TextField'
def get_db_prep_lookup(self, lookup_type, value):
if lookup_type not in ['exact', 'in', 'isnull']:
raise TypeError('Lookup type %s is not supported.' % lookup_type)
return super(PickledObjectField, self).get_db_prep_lookup(lookup_type, value)