Develop Reference

How to do left outer join with PeeWee and no ForeignKey?

Using PeeWee on top of SQLite, I am trying to do a left outer join between two tables that do not have a ForeignKey relation defined. I can get the data if the right table an entry that matches the left table, but if there is no match, the columns in the right table do not make it into the returned models.
class BaseModel(Model):
class Meta:
database = db
class Location(BaseModel):
location_key = CharField(primary_key=True)
lat = FloatField(null = False)
lon = FloatField(null = False)
class Household(BaseModel):
name = CharField(null=True)
location_id = CharField(null=True)
I am trying to do something like:
for h in Household.select(Household,Location).join(Location, on=(Household.location_id == Location.location_key), join_type=JOIN.LEFT_OUTER):
print(type(h), h, h.location, h.location.lat)
This works if Household.location_id matches something in Location, but if Household.location_id is None (null), then I get an AttributeError: 'Household' object has no attribute 'location'
I would have expected location to be present, but have a valid of None.
How can I check for the existence of location before using it? I am trying to avoid using ForeignKey, there are a lot of mismatches between Household.location_id and Location.location_key and PeeWee really gets angry about that...

I think I understand what you're trying to do after re-reading. What I'd suggest is to use Peewee's "on" keyword argument in the join, which can patch the related Location (if it exists) onto a different attr than "location":
query = (HouseHold
.select(HouseHold, Location)
.join(Location, on=(HouseHold.location_id == Location.location_key),
attr='location_obj', join_type=JOIN.LEFT_OUTER))
Then you can check the "location_obj" to retrieve the related object.
for house in query:
# if there was a match, get the location obj or None.
location_obj = getattr(house, 'location_obj', None)
# the location_id is still present.
print(house.location_id, location_obj)

Found my own answer. Implement __getattr__(self) in the Household model, and return None if the name is 'location'. __getattr__(self) is only called if there is no property with that name.

Hybrid property expression with JOIN

I'm fairly new to peewee, but have some strong background on SQLAlchemy (and all the vices that come with it). I'm trying to create a custom hybrid expression that correlates to a third (or even N) table. I'll try to demonstrate in an example (non-tested) code:
class BaseModel(Model):
class Meta:
database = database
class Person(BaseModel):
id = PrimaryKeyField(column_name="person_id")
name = CharField(max_length=255, column_name="person_name")
username = CharField(max_length=255, column_name="person_username")
class PersonTree(BaseModel):
id = PrimaryKeyField(column_name="person_tree_id")
name = CharField(max_length=255, column_name="person_tree_name")
code = CharField(max_length=255, column_name="person_tree_code")
person = ForeignKeyField(
column_name="person_id",
model=Person,
field="id",
backref="tree",
)
class Article(BaseModel):
id = PrimaryKeyField(column_name="article_id")
name = CharField(max_length=255, column_name="article_name")
branch = ForeignKeyField(
column_name="person_tree_id",
model=PersonTree,
field="id",
backref="articles",
)
#hybrid_property
def username(self):
"""
This gives me the possibility to grab the direct username of an article
"""
return self.branch.person.username
#username.expression
def username(cls):
"""
What if I wanted to do: Article.query().where(Article.username == "john_doe") ?
"""
pass
With the username hybrid_property on Article, I can get the username of the Person related to an Article using the PersonTree as a correlation, so far so good, but ... What if I wanted to "create a shortcut" to query all Articles created by the "john_doe" Person username, without declaring the JOINs every time I make the query and without relying on .filter(branch__person__username="john_doe")? I know it's possible with SA (to a great extent), but I'm finding this hard to accomplish with peewee.
Just for clarification, here's the SQL I hope to be able to construct:
SELECT
*
FROM
article a
JOIN person_tree pt ON a.person_tree_id = pt.person_tree_id
JOIN person p ON pt.person_id = p.person_id
WHERE
p.username = 'john_doe';
Thanks a lot in advance!

Hybrid properties can be used to allow an attribute to be expressed as a property of a model instance or as a scalar computation in a SQL query.
What you're trying to do, which is add multiple joins and stuff via the property, is not possible using hybrid properties.
What if I wanted to "create a shortcut" to query all Articles created by the "john_doe" Person username
Just add a normal method:
#classmethod
def by_username(cls, username):
return (Article
.select(Article, PersonTree, Person)
.join(PersonTree)
.join(Person)
.where(Person.name == username))

Orator ORM model Create method invalid SQL

I have a database I created with a migration. One of my tables looks like this
def create_customer_table(self):
with self.schema.create("customer") as table:
table.char("name",120).unique()
table.integer("transmitting_hours").default(24) #how many hours after transmission vehicle is considered transmitting
table.boolean("is_tpms").default(False)
table.boolean("is_dor").default(False)
table.boolean("is_otr").default(False)
table.boolean("is_track_and_trace").default(False)
table.char("contact_person",25)
table.char("created_by",25)
table.enum("temperature_unit",TEMP_UNITS)
table.enum("pressure_unit",PRESSURE_UNITS)
table.enum("distance_unit",DISTANCE_UNITS)
table.char("time_zone",25)
table.char("language",2)
table.timestamps()
I have a very simplistic ORM model on top
class Customer(Model):
__table__ = "customer"
__timestamps__ = False
__primary_key__ = "name"
__fillable__ = ['*']
I then try to do a basic insert with the following code
def add_sample_customer():
sample_customer = {}
sample_customer["name"] = "customer_2"
sample_customer["contact_person"] = "Abradolf"
sample_customer["created_by"] = "Frodo"
sample_customer["time_zone"] = "GMT-5"
sample_customer["language"] = "EN"
sample_customer["temperature_unit"] = "FAHRENHEIT"
sample_customer["pressure_unit"] = "PSI"
sample_customer["distance_unit"] = "MI"
customer_model = Customer.create(_attributes = sample_customer)
The exception I get from this code looks like
orator.exceptions.query.QueryException: syntax error at or near ")"
LINE 1: INSERT INTO "customer" () VALUES () RETURNING "name"
(SQL: INSERT INTO "customer" () VALUES () RETURNING "name" ([]))
it looks like orator just isn't filling in the cols and vals here. I have also tried it with a few different syntactic ways of dropping the dict in there, using **sample_customer and also just putting the dict in directly and none of them work, all with the same exception. I started debugging by printing stuff out of the orator libraries but haven't gotten anywhere yet.
my inserts work if I do the model attribute assignment individually and use the model.save() method like this
def add_sample_customer():
sample_customer = {}
sample_customer["name"] = "customer_2"
sample_customer["contact_person"] = "Abradolf"
sample_customer["created_by"] = "Frodo"
sample_customer["time_zone"] = "GMT-5"
sample_customer["language"] = "EN"
sample_customer["temperature_unit"] = "FAHRENHEIT"
sample_customer["pressure_unit"] = "PSI"
sample_customer["distance_unit"] = "MI"
customer_model = Customer()
for k,v in sample_customer.items():
setattr(customer_model,k,v)
customer_model.save()
Does anyone understand why the model.create() syntax fails?

I would think the answer would be:
Simply passing the dictionary instead of using keyword notation with attributes:
Customer.create(sample_customer)
or
Customer.create(attribute=value,attribute2=value2,..etc)
Which are the valid notations

SQLalchemy find id and use it to lookup other information

I'm making a simple lookup application for Japanese characters (Kanji), where the user can search the database using any of the information available.
My database structure
Kanji:
id
character (A kanji like 頑)
heisig6 (a number indicating the order of showing Kanji)
kanjiorigin (a number indicating the order of showing Kanji)
MeaningEN (1 kanji_id can have multiple entries with different meanings):
kanji_id (FOREIGN KEY(kanji_id) REFERENCES "Kanji" (id)
meaning
User handling
The user can choose to search by 'id', 'character', 'heisig6', 'kanjiorigin' or 'meaning' and it should then return all information in all those fields. (All fields return only 1 result, except meanings, which can return multiple results)
Code, EDIT 4+5: my code with thanks to #ApolloFortyNine and #sqlalchemy on IRC, EDIT 6: join --> outerjoin (otherwise won't find information that has no Origins)
import sqlalchemy as sqla
import sqlalchemy.orm as sqlo
from tableclass import TableKanji, TableMeaningEN, TableMisc, TableOriginKanji # See tableclass.py
# Searches database with argument search method
class SearchDatabase():
def __init__(self):
#self.db_name = "sqlite:///Kanji_story.db"
self.engine = sqla.create_engine("sqlite:///Kanji.db", echo=True)
# Bind the engine to the metadata of the Base class so that the
# declaratives can be accessed through a DBSession instance
tc.sqla_base.metadata.bind = self.engine
# For making sessions to connect to db
self.db_session = sqlo.sessionmaker(bind=self.engine)
def retrieve(self, s_input, s_method):
# s_input: search input
# s_method: search method
print("\nRetrieving results with input: {} and method: {}".format(s_input, s_method))
data = [] # Data to return
# User searches on non-empty string
if s_input:
session = self.db_session()
# Find id in other table than Kanji
if s_method == 'meaning':
s_table = TableMeaningEN # 'MeaningEN'
elif s_method == 'okanji':
s_table = TableOriginKanji # 'OriginKanji'
else:
s_table = TableKanji # 'Kanji'
result = session.query(TableKanji).outerjoin(TableMeaningEN).outerjoin(
(TableOriginKanji, TableKanji.origin_kanji)
).filter(getattr(s_table, s_method) == s_input).all()
print("result: {}".format(result))
for r in result:
print("r: {}".format(r))
meanings = [m.meaning for m in r.meaning_en]
print(meanings)
# TODO transform into origin kanji's
origins = [str(o.okanji_id) for o in r.okanji_id]
print(origins)
data.append({'character': r.character, 'meanings': meanings,
'indexes': [r.id, r.heisig6, r.kanjiorigin], 'origins': origins})
session.close()
if not data:
data = [{'character': 'X', 'meanings': ['invalid', 'search', 'result']}]
return(data)
Question EDIT 4+5
Is this an efficient query?: result = session.query(TableKanji).join(TableMeaningEN).filter(getattr(s_table, s_method) == s_input).all() (The .join statement is necessary, because otherwise e.g. session.query(TableKanji).filter(TableMeaningEN.meaning == 'love').all() returns all the meanings in my database for some reason? So is this either the right query or is my relationship() in my tableclass.py not properly defined?
fixed (see lambda: in tableclass.py) kanji = relationship("TableKanji", foreign_keys=[kanji_id], back_populates="OriginKanji") <-- what is wrong about this? It gives the error:
File "/path/python3.5/site-packages/sqlalchemy/orm/mapper.py", line 1805, in get_property
"Mapper '%s' has no property '%s'" % (self, key))
sqlalchemy.exc.InvalidRequestError: Mapper 'Mapper|TableKanji|Kanji' has no property 'OriginKanji'
Edit 2: tableclass.py (EDIT 3+4+5: updated)
import sqlalchemy as sqla
from sqlalchemy.orm import relationship
import sqlalchemy.ext.declarative as sqld
sqla_base = sqld.declarative_base()
class TableKanji(sqla_base):
__tablename__ = 'Kanji'
id = sqla.Column(sqla.Integer, primary_key=True)
character = sqla.Column(sqla.String, nullable=False)
radical = sqla.Column(sqla.Integer) # Can be defined as Boolean
heisig6 = sqla.Column(sqla.Integer, unique=True, nullable=True)
kanjiorigin = sqla.Column(sqla.Integer, unique=True, nullable=True)
cjk = sqla.Column(sqla.String, unique=True, nullable=True)
meaning_en = relationship("TableMeaningEN", back_populates="kanji") # backref="Kanji")
okanji_id = relationship("TableOriginKanji", foreign_keys=lambda: TableOriginKanji.kanji_id, back_populates="kanji")
class TableMeaningEN(sqla_base):
__tablename__ = 'MeaningEN'
kanji_id = sqla.Column(sqla.Integer, sqla.ForeignKey('Kanji.id'), primary_key=True)
meaning = sqla.Column(sqla.String, primary_key=True)
kanji = relationship("TableKanji", back_populates="meaning_en")
class TableOriginKanji(sqla_base):
__tablename__ = 'OriginKanji'
kanji_id = sqla.Column(sqla.Integer, sqla.ForeignKey('Kanji.id'), primary_key=True)
okanji_id = sqla.Column(sqla.Integer, sqla.ForeignKey('Kanji.id'), primary_key=True)
order = sqla.Column(sqla.Integer)
#okanji = relationship("TableKanji", foreign_keys=[kanji_id], backref="okanji")
kanji = relationship("TableKanji", foreign_keys=[kanji_id], back_populates="okanji_id")

We would really have to be able to see your database schema to give real critique, but assuming no foreign keys, what you said is basically the best you can do.
SQLAlchemy really begins to shine when you have complicated relations going on however. For example, if you properly had foreign keys set, you could do something like the following.
# Assuming kanji is a tc.tableMeaningEN.kanji_id object
kanji_meaning = kanji.meanings
And that would return the meanings for the kanji as an array, without any further queries.
You can go quite deep with relationships, so I'm linking the documentation here. http://docs.sqlalchemy.org/en/latest/orm/relationships.html
EDIT: Actually, you don't need to manually join at all, SQLAlchemy will do it for you.
The case is wrong on your classes, but I'm not sure if SQLAlchemy is case sensitive there or not. If it works, then just move on.
If you query the a table (self.session.query(User).filter(User.username == self.name).first()) you should have an object of the table type (User here).
So in your case, querying the TableKanji table alone will return an object of that type.
kanji_obj = session.query(TableKanji).filter(TableKanji.id == id).first()
# This will return an array of all meaning_ens that match the foreign key
meaning_arr = kanji_obj.meaning_en
# This will return a single meeting, just to show each member of the arr is of type TableMeaningEn
meaning_arr[0].meaning
I have a project made use of some of these features, hope it helps:
https://github.com/ApolloFortyNine/SongSense
Database declaration (with relationships): https://github.com/ApolloFortyNine/SongSense/blob/master/songsense/database.py
Automatic joins: https://github.com/ApolloFortyNine/SongSense/blob/master/songsense/getfriend.py#L134
I really like my database structure, but as for the rest it's pretty awful. Hope it still helps though.

How would you inherit from and override the django model classes to create a listOfStringsField?

I want to create a new type of field for django models that is basically a ListOfStrings. So in your model code you would have the following:
models.py:
from django.db import models
class ListOfStringsField(???):
???
class myDjangoModelClass():
myName = models.CharField(max_length=64)
myFriends = ListOfStringsField() #
other.py:
myclass = myDjangoModelClass()
myclass.myName = "bob"
myclass.myFriends = ["me", "myself", "and I"]
myclass.save()
id = myclass.id
loadedmyclass = myDjangoModelClass.objects.filter(id__exact=id)
myFriendsList = loadedclass.myFriends
# myFriendsList is a list and should equal ["me", "myself", "and I"]
How would you go about writing this field type, with the following stipulations?
We don't want to do create a field which just crams all the strings together and separates them with a token in one field like this. It is a good solution in some cases, but we want to keep the string data normalized so tools other than django can query the data.
The field should automatically create any secondary tables needed to store the string data.
The secondary table should ideally have only one copy of each unique string. This is optional, but would be nice to have.
Looking in the Django code it looks like I would want to do something similar to what ForeignKey is doing, but the documentation is sparse.
This leads to the following questions:
Can this be done?
Has it been done (and if so where)?
Is there any documentation on Django about how to extend and override their model classes, specifically their relationship classes? I have not seen a lot of documentation on that aspect of their code, but there is this.
This is comes from this question.

There's some very good documentation on creating custom fields here.
However, I think you're overthinking this. It sounds like you actually just want a standard foreign key, but with the additional ability to retrieve all the elements as a single list. So the easiest thing would be to just use a ForeignKey, and define a get_myfield_as_list method on the model:
class Friends(model.Model):
name = models.CharField(max_length=100)
my_items = models.ForeignKey(MyModel)
class MyModel(models.Model):
...
def get_my_friends_as_list(self):
return ', '.join(self.friends_set.values_list('name', flat=True))
Now calling get_my_friends_as_list() on an instance of MyModel will return you a list of strings, as required.

What you have described sounds to me really similar to the tags.
So, why not using django tagging?
It works like a charm, you can install it independently from your application and its API is quite easy to use.

I also think you're going about this the wrong way. Trying to make a Django field create an ancillary database table is almost certainly the wrong approach. It would be very difficult to do, and would likely confuse third party developers if you are trying to make your solution generally useful.
If you're trying to store a denormalized blob of data in a single column, I'd take an approach similar to the one you linked to, serializing the Python data structure and storing it in a TextField. If you want tools other than Django to be able to operate on the data then you can serialize to JSON (or some other format that has wide language support):
from django.db import models
from django.utils import simplejson
class JSONDataField(models.TextField):
__metaclass__ = models.SubfieldBase
def to_python(self, value):
if value is None:
return None
if not isinstance(value, basestring):
return value
return simplejson.loads(value)
def get_db_prep_save(self, value):
if value is None:
return None
return simplejson.dumps(value)
If you just want a django Manager-like descriptor that lets you operate on a list of strings associated with a model then you can manually create a join table and use a descriptor to manage the relationship. It's not exactly what you need, but this code should get you started.

Thanks for all those that answered. Even if I didn't use your answer directly the examples and links got me going in the right direction.
I am not sure if this is production ready, but it appears to be working in all my tests so far.
class ListValueDescriptor(object):
def __init__(self, lvd_parent, lvd_model_name, lvd_value_type, lvd_unique, **kwargs):
"""
This descriptor object acts like a django field, but it will accept
a list of values, instead a single value.
For example:
# define our model
class Person(models.Model):
name = models.CharField(max_length=120)
friends = ListValueDescriptor("Person", "Friend", "CharField", True, max_length=120)
# Later in the code we can do this
p = Person("John")
p.save() # we have to have an id
p.friends = ["Jerry", "Jimmy", "Jamail"]
...
p = Person.objects.get(name="John")
friends = p.friends
# and now friends is a list.
lvd_parent - The name of our parent class
lvd_model_name - The name of our new model
lvd_value_type - The value type of the value in our new model
This has to be the name of one of the valid django
model field types such as 'CharField', 'FloatField',
or a valid custom field name.
lvd_unique - Set this to true if you want the values in the list to
be unique in the table they are stored in. For
example if you are storing a list of strings and
the strings are always "foo", "bar", and "baz", your
data table would only have those three strings listed in
it in the database.
kwargs - These are passed to the value field.
"""
self.related_set_name = lvd_model_name.lower() + "_set"
self.model_name = lvd_model_name
self.parent = lvd_parent
self.unique = lvd_unique
# only set this to true if they have not already set it.
# this helps speed up the searchs when unique is true.
kwargs['db_index'] = kwargs.get('db_index', True)
filter = ["lvd_parent", "lvd_model_name", "lvd_value_type", "lvd_unique"]
evalStr = """class %s (models.Model):\n""" % (self.model_name)
evalStr += """ value = models.%s(""" % (lvd_value_type)
evalStr += self._params_from_kwargs(filter, **kwargs)
evalStr += ")\n"
if self.unique:
evalStr += """ parent = models.ManyToManyField('%s')\n""" % (self.parent)
else:
evalStr += """ parent = models.ForeignKey('%s')\n""" % (self.parent)
evalStr += "\n"
evalStr += """self.innerClass = %s\n""" % (self.model_name)
print evalStr
exec (evalStr) # build the inner class
def __get__(self, instance, owner):
value_set = instance.__getattribute__(self.related_set_name)
l = []
for x in value_set.all():
l.append(x.value)
return l
def __set__(self, instance, values):
value_set = instance.__getattribute__(self.related_set_name)
for x in values:
value_set.add(self._get_or_create_value(x))
def __delete__(self, instance):
pass # I should probably try and do something here.
def _get_or_create_value(self, x):
if self.unique:
# Try and find an existing value
try:
return self.innerClass.objects.get(value=x)
except django.core.exceptions.ObjectDoesNotExist:
pass
v = self.innerClass(value=x)
v.save() # we have to save to create the id.
return v
def _params_from_kwargs(self, filter, **kwargs):
"""Given a dictionary of arguments, build a string which
represents it as a parameter list, and filter out any
keywords in filter."""
params = ""
for key in kwargs:
if key not in filter:
value = kwargs[key]
params += "%s=%s, " % (key, value.__repr__())
return params[:-2] # chop off the last ', '
class Person(models.Model):
name = models.CharField(max_length=120)
friends = ListValueDescriptor("Person", "Friend", "CharField", True, max_length=120)
Ultimately I think this would still be better if it were pushed deeper into the django code and worked more like the ManyToManyField or the ForeignKey.

I think what you want is a custom model field.