Develop Reference

How to make (correctly) runtime database connection in python telegram bot?

I'm doing my first telegram-bot-project using python+peewee+postgresql (without django).
I just want to know, how to connect to my database not once (in the start of my project's code), but everytime when it's needed. For example: user tap on button -> connection opens -> it adds line to table(s)
Right now file structure looks like this:
tgbot.py
# some lines with imports
updater = Updater(TELEGRAM_TOKEN)
dp = updater.dispatcher
dp = setup_dispatcher(dp)
# some lines with logging config
updater.start_polling()
updater.idle()
dispatcher.py
# some lines with imports
# some def for event handlers
# !!!!! here will be database connections (in functions for events)
def setup_dispatcher(dp):
db_conn = DBConnection() # Enter database connection
from dbmodels import db
db = db_conn.get_connection()
ToDo.create(...) # creating line in postgres-table
dp.add_handler(CommandHandler('start', start))
dp.add_handler(CommandHandler('location', ask_for_location))
dp.add_handler(MessageHandler(Filters.location, change_location))
dp.add_handler(CommandHandler('today', ask_for_today))
return dp
dbhelper.py
# some lines with imports
class DBConnection(Singleton):
def __init__(self): ... # initializing some variables like self.database, etc
def get_connection(self):
""" Creating PostgreSQL's database connection """
if self.connection is None:
try:
self.connection = PostgresqlDatabase(self.database, user=self.user, password=self.password, host=self.host, port=self.port)
self.curs = self.connection.cursor()
except (Exception, Error) as error:
print("PostgreSQL's connection error: \n", error)
sys.exit(1)
return self.connection
def __del__(self):
""" Closing database connection """
if self.connection is not None:
self.curs.close()
self.connection.close()
print("PostgreSQL's connection closed.")
dbmodels.py
# some lines with imports
db = PostgresqlDatabase(None)
class Singleton:
""" Singleton realisation for database connection class in dbhelper.py """
_instance = None
def __new__(cls):
if cls._instance is None:
cls._instance = super().__new__(cls)
return cls._instance
class BaseModel(Model):
""" Basic class, used for tables-classes in models.py """
class Meta:
database = db
models.py
# some lines with imports
class User(BaseModel):
""" A model for client's table """
# initializing some fields
class Meta:
db_table = "clients"
...
class ToDo(BaseModel):
""" A model of to-do's table """
# initializing some fields
class Meta:
db_table = "to_do_s"
...

This is all completely unnecessary. The peewee database object is already a singleton. You just need to call the connect() and close() methods on it when you want to connect/close.
Alternatively you can use the database instance as a context-manager, e.g.:
db = PostgresqlDatabase(...)
with db:
ToDo.create(...)

Create database for unit testing in sqlite

I have a simple class driven sqlite app. Basically, I want to run unit tests for it, but I haven't been able to so far.
class DB:
def __init__(self, dbname='mydb.db'):
try:
self.connection = sqlite3.connect(dbname)
except:
print('Error')
finally:
pass
Any class would use it then:
class Hello:
def hi(self):
db = DB() # Create db or connect to existing one
cursor = db.connection.cursor()
Now, when testing, I pass on a test database:
db = DB('test.db')
#create datatabase here and all works fine
h = Hello()
Now, h makes use of mydb.db, instead of test.db. How does one go out about testing the above structure?

If you want to pass the instance of your DB class (db) you would need to feed the instance to your Hello class. Try:
class DB:
def __init__(self, dbname='mydb.db'):
try:
self.connection = sqlite3.connect(dbname)
except:
print('Error')
finally:
pass
class Hello:
def hi(self, db=DB()): # we make it have a default db of DB() (which in turn defaults to 'mydb.db')
# db = DB() # Create db or connect to existing one
# ^^ we remove this line because now this is the default
cursor = db.connection.cursor()
db = DB('test.db') # this makes an instance of your DB class and calls it "db"
h = Hello(db) # now we need to feed this instance
Although this probably isn't the best way to go about it. You would likely benefit more from having a single class with methods because your second class is basically useless and is very closely related to your first class anyways:
class DB:
def __init__(self, dbname='mydb.db'):
try:
self.connection = sqlite3.connect(dbname)
except:
print('Error')
finally:
pass
def hello(self): # making a method instead
cursor = self.connection.cursor()
db = DB('test.db') # this makes an instance of your DB class and calls it "db"
db.hello() # call our method
EDIT
I missed something originally that I found from testing my code. Your code should work fine, but you need to call the method you've made! Try this:
import sqlite3
class DB:
def __init__(self, dbname='mydb.db'):
try:
self.connection = sqlite3.connect(dbname)
except:
print('Error')
finally:
pass
class Hello:
def hi(self):
db = DB('test.db')
cursor = db.connection.cursor()
db = DB('test.db')
h = Hello() # make our instance
h.hi() # use the method "hi" associated with the class (our function name within the class)

Pass database parameter into peewee Meta class

I am using peewee to manage CRUD operations on a Postgres database.
In the project documentation, connection to the database and creation of the ORM should be setup through a Meta class which other ORM types should inherit.
from peewee import *
db = PostgresqlDatabase('table', **{})
class BaseModel(Model):
class Meta:
database = db
class Product(BaseModel):
name = CharField(unique=True)
I'd like to be able to encapsulate this setup in a Persistence class, like as follows (so as not to create any global variables):
class Persistence():
db = None
class BaseModel(Model):
class Meta:
database = Persistence.db
class Product(BaseModel):
name = CharField(unique=True)
def __init__(self):
self.db = PostgresqlDatabase('table', **{})
Unfortunately this doesn't work with:
AttributeError: type object 'Persistence' has no attribute 'db'
I don't think this would work as expected (disregarding the AttributeError), because even if the variable was in scope at the time of the creation of BaseModel, it would be None and not change when the Persistence class is instantiated.
Is there a way to scope the db variable correctly so that it uses a class attribute on Persistence?
Can I pass in this db connection to peewee through another mechanism?

Your issue confuses two separate issues. Python scoping and Peewee database initialization. It'd be helpful if you were clear on where exactly the issue lies.
For the peewee part of your question, you probably want to defer initialization of the database. To do this, you create a database object placeholder -- or use a Proxy depending on how late you want to defer things.
Example of deferred initialization:
db = SqliteDatabase(None)
class BaseModel(Model):
class Meta:
database = db
# Declare other models as subclasses of BaseModel, e.g.
class Foo(BaseModel):
data = TextField()
class Persistence(object):
def __init__(self, db_file):
db.init(db_file)
Alternatively you can use a proxy, which is documented here: http://docs.peewee-orm.com/en/latest/peewee/database.html#dynamically-defining-a-database

You can make use of 'Using' execution contexts to have dynamic db connections. For example I have something like this:
from peewee import *
from playhouse.shortcuts import RetryOperationalError
from config import settings
class RetryMySQLDatabase(RetryOperationalError, MySQLDatabase):
pass
db_connection = {}
for conn in settings.DB:
dbs = settings.DB[conn]
db_connection[conn] = RetryMySQLDatabase(
dbs["DB_NAME"],
host=dbs["DB_HOST"],
user=dbs["DB_USER"],
password=dbs["DB_PASS"]
)
db = db_connection["default"] #if you don't want to have default connection, you can make use of Proxy() here
class BaseModel(Model):
class Meta:
database = db
class Booking(BaseModel):
id = BigIntegerField(db_column='ID', primary_key=True)
name = CharField(db_column='NAME', null=True)
And, while using the model class you can specify which db connection to use:
with Using(db_connection["read_only"], [Booking]):
booking_data = Booking.get(Booking.id == 123)
Ref:
'Using' execution context: http://docs.peewee-orm.com/en/2.10.2/peewee/database.html?highlight=re#using-multiple-databases
Proxy class: http://docs.peewee-orm.com/en/latest/peewee/database.html#dynamically-defining-a-database

How to manage a peewee database in a separate module?

I want to have my database implementation in a separate module or class. But I am struggling with a few details. A simple example:
from peewee import *
db = SqliteDatabase(':memory:')
class BaseModel(Model):
class Meta:
database = db
class User(BaseModel):
name = CharField()
db.connect()
db.create_tables([User,])
db.commit()
#db.atomic()
def add_user(name):
User.create(name=name).save()
#db.atomic()
def get_user(name):
return User.get(User.name == name)
So far this is working fine. I can implement my interface to the database here and import this as a module.
Now I want to be able to choose the database file at runtime. So I need a way to define the Model classes without defining SqliteDatabase('somefile') before. I tried to encapsulate everything in a new Database class, which I can later import and create an instance from:
from peewee import *
class Database:
def __init__(self, dbfile):
self.db = SqliteDatabase(dbfile)
class BaseModel(Model):
class Meta:
database = self.db
class User(BaseModel):
name = CharField()
self.User = User
self.db.connect()
self.db.create_tables([User,])
self.db.commit()
#self.db.atomic() # Error as self is not known on this level
def add_user(self, name):
self.User.create(name=name).save()
#self.db.atomic() # Error as self is not known on this level
def get_user(self, name):
return self.User.get(self.User.name == name)
Now I can call for example database = Database('database.db') or choose any other file name. I can even use multiple database instance in the same program, each with its own file.
However, there are two problems with this approach:
I still need to specify the database driver (SqliteDatabase) before defining the Model classes. To solve this I define the Model classes within the __init__() method, and then create an alias to with self.User = User. I don't really like this approach (it just doesn't feel like neat code), but at least it works.
I cannot use the #db.atomic() decorator since self is not known at class level, I would an instance here.
So this class approach does not seem to work very well. Is there some better way to define the Model classes without having to choose where you want to store your database first?

If you need to change database driver at the runtime, then Proxy is a way to go
# database.py
import peewee as pw
proxy = pw.Proxy()
class BaseModel(pw.Model):
class Meta:
database = proxy
class User(BaseModel):
name = pw.CharField()
def add_user(name):
with proxy.atomic() as txn:
User.create(name=name).save()
def get_user(name):
with proxy.atomic() as txn:
return User.get(User.name == name)
From now on each time you load the module, it won't need a database to be initialized. Instead, you can initialize it at the runtime and switch between multiple as follows
# main.py
import peewee as pw
import database as db
sqlite_1 = pw.SqliteDatabase('sqlite_1.db')
sqlite_2 = pw.PostgresqlDatabase('sqlite_2.db')
db.proxy.initialize(sqlite_1)
sqlite_1.create_tables([db.User], safe=True)
db.add_user(name="Tom")
db.proxy.initialize(sqlite_2)
sqlite_2.create_tables([db.User], safe=True)
db.add_user(name="Jerry")
But if the connection is the only thing that matters, then init() method will be enough.

Now I want to be able to choose the database file at runtime. So I
need a way to define the Model classes without defining
SqliteDatabase('somefile') before. I tried to encapsulate everything
in a new Database class, which I can later import and create an
instance from
Peewee uses the meta class to define the name of the table (Model.Meta.db_table) and database( Model.Meta.database)
set these attribute before calling a Model specific code ( either to create a table or to DML statements)
'Allow to define database dynamically'

Question: I cannot use the #db.atomic() decorator since self is not known at class level
Do it, as you do it with self.User.
I wonder about atomic() instead of atomic, but you tell is working fine.
class Database:
def __init__(self, dbfile):
self.db = SqliteDatabase(dbfile)
...
#self.db.atomic()
def __add_user(self, name):
self.User.create(name=name).save()
self.add_user = __add_user
#self.db.atomic()
def __get_user(self, name):
return self.User.get(self.User.name == name)
self.get_user = __get_user
Related: Define models separately from Database() initialization

How to create table during Django tests with managed = False

I have a model with managed = False.
class SampleModel(models.Model):
apple = models.CharField(max_length=30)
orange = models.CharField(max_length=30)
class Meta:
managed = False
I have a unit test which creates a SampleModel, however when I run the test I get:
DatabaseError: no such table: SAMPLE_SAMPLE_MODEL
The django docs - https://docs.djangoproject.com/en/dev/ref/models/options/#managed documents the following:
For tests involving models with managed=False, it's up to you to
ensure the correct tables are created as part of the test setup.
How can I actually "create" the tables during the test setup? Or alternatively, how can I make it so that when I am running tests, this model has "managed = True" for the duration of the test?
In the real application, this model is actually backed by a view in the database. However for the during of the test, I would like to treat this as a table and be able to insert test data in there.

You can use SchemaEditor in TestCase.setUp method to explicitly create models with managed = False.
# models.py
from django.db import models
class Unmanaged(models.Model):
foo = models.TextField()
class Meta:
# This model is not managed by Django
managed = False
db_table = 'unmanaged_table'
And in your tests:
# tests.py
from django.db import connection
from django.test import TestCase
from myapp.models import Unmanaged
class ModelsTestCase(TestCase):
def setUp(self):
super().setUp()
with connection.schema_editor() as schema_editor:
schema_editor.create_model(Unmanaged)
if (
Unmanaged._meta.db_table
not in connection.introspection.table_names()
):
raise ValueError(
"Table `{table_name}` is missing in test database.".format(
table_name=Unmanaged._meta.db_table
)
)
def tearDown(self):
super().tearDown()
with connection.schema_editor() as schema_editor:
schema_editor.delete_model(Unmanaged)
def test_unmanaged_model(self):
with self.assertNumQueries(num=3):
self.assertEqual(0, Unmanaged.objects.all().count())
Unmanaged.objects.create()
self.assertEqual(1, Unmanaged.objects.all().count())

Check out this blog post: http://www.caktusgroup.com/blog/2010/09/24/simplifying-the-testing-of-unmanaged-database-models-in-django/ It describes in detail the creation of a test runner for unmanaged models.
from django.test.simple import DjangoTestSuiteRunner
class ManagedModelTestRunner(DjangoTestSuiteRunner):
"""
Test runner that automatically makes all unmanaged models in your Django
project managed for the duration of the test run, so that one doesn't need
to execute the SQL manually to create them.
"""
def setup_test_environment(self, *args, **kwargs):
from django.db.models.loading import get_models
self.unmanaged_models = [m for m in get_models()
if not m._meta.managed]
for m in self.unmanaged_models:
m._meta.managed = True
super(ManagedModelTestRunner, self).setup_test_environment(*args,
**kwargs)
def teardown_test_environment(self, *args, **kwargs):
super(ManagedModelTestRunner, self).teardown_test_environment(*args,
**kwargs)
# reset unmanaged models
for m in self.unmanaged_models:
m._meta.managed = False

Execute raw SQL to create the table in the test setup:
from django.db import connection
class MyTest(unittest.TestCase):
def setUp(self):
connection.cursor().execute("CREATE TABLE ...")
def tearDown(self):
connection.cursor().execute("DROP TABLE ...")

Nice plug and play solution. Just paste this before your test class definition. (note: django 1.8 used)
from django.db.models.loading import get_models
def change_managed_settings_just_for_tests():
"""django model managed bit needs to be switched for tests."""
unmanaged_models = [m for m in get_models() if not m._meta.managed]
for m in unmanaged_models:
m._meta.managed = True
change_managed_settings_just_for_tests()

A quick fix if you don't have many unmanaged tables:
First add a new variable to the settings.
# settings.py
import sys
UNDER_TEST = (len(sys.argv) > 1 and sys.argv[1] == 'test')
then in the models
# models.py
from django.conf import settings
class SampleModel(models.Model):
apple = models.CharField(max_length=30)
orange = models.CharField(max_length=30)
class Meta:
managed = getattr(settings, 'UNDER_TEST', False)

Create your own test runner using this:
from django.test.simple import DjangoTestSuiteRunner
class NoDbTestRunner(DjangoTestSuiteRunner):
""" A test runner to test without database creation """
def setup_databases(self, **kwargs):
""" Override the database creation defined in parent class """
#set manage=True for that specific database on here
Then on your settings add this class to TEST_RUNNER.

Just to add :django.db.models.loading.get_models will be removed in Django 1.9 (see https://github.com/BertrandBordage/django-cachalot/issues/33).
Below is an updated one for Django 1.10:
class UnManagedModelTestRunner(DiscoverRunner):
'''
Test runner that automatically makes all unmanaged models in your Django
project managed for the duration of the test run.
Many thanks to the Caktus Group
'''
def setup_test_environment(self, *args, **kwargs):
from django.apps import apps
self.unmanaged_models = [m for m in apps.get_models() if not m._meta.managed]
for m in self.unmanaged_models:
m._meta.managed = True
super(UnManagedModelTestRunner, self).setup_test_environment(*args, **kwargs)
def teardown_test_environment(self, *args, **kwargs):
super(UnManagedModelTestRunner, self).teardown_test_environment(*args, **kwargs)
# reset unmanaged models
for m in self.unmanaged_models:
m._meta.managed = False
Note you also need to take care migrations(see Testing django application with several legacy databases)
MIGRATION_MODULES = {
'news': 'news.test_migrations',
'economist': 'economist.test_migrations'
}

Using pytest and pytest-django
To make this work (has been tested with django 3.0.2, pytest 5.3.5 and pytest-django 3.8.0):
Run your pytest with the additional argument --no-migrations.
Put the following code in your conftest.py. This has an unfortunate amount of copypasta, but I could not figure out how to first make my models unmanaged and then call the original django_db_setup. The issue of not being able to call pytest fixtures directly is discussed here: https://github.com/pytest-dev/pytest/issues/3950
conftest.py
# example file
import pytest
from pytest_django.fixtures import _disable_native_migrations
#pytest.fixture(scope="session")
def django_db_setup(
request,
django_test_environment,
django_db_blocker,
django_db_use_migrations,
django_db_keepdb,
django_db_createdb,
django_db_modify_db_settings,
):
# make unmanaged models managed
from django.apps import apps
unmanaged_models = []
for app in apps.get_app_configs():
unmanaged_models = [m for m in app.get_models()
if not m._meta.managed]
for m in unmanaged_models:
m._meta.managed = True
# copypasta fixture code
"""Top level fixture to ensure test databases are available"""
from pytest_django.compat import setup_databases, teardown_databases
setup_databases_args = {}
if not django_db_use_migrations:
_disable_native_migrations()
if django_db_keepdb and not django_db_createdb:
setup_databases_args["keepdb"] = True
with django_db_blocker.unblock():
db_cfg = setup_databases(
verbosity=request.config.option.verbose,
interactive=False,
**setup_databases_args
)
def teardown_database():
with django_db_blocker.unblock():
try:
teardown_databases(db_cfg, verbosity=request.config.option.verbose)
except Exception as exc:
request.node.warn(
pytest.PytestWarning(
"Error when trying to teardown test databases: %r" % exc
)
)
if not django_db_keepdb:
request.addfinalizer(teardown_database)

After spending a few hours testing and researching ways to test my django unmanaged models, I finally came up with a solution that worked for me.
My implementation is in this below snippet. It's working great with local tests using db.sqlite3.
# Example classes Implementation
from django.db import models, connection
from django.db.models.base import ModelBase as DjangoModelBase
from django.db.utils import OperationalError
from django.test import TransactionTestCase
class AbstractModel(models.Model):
name = models.TextField(db_column="FIELD_NAME", blank=True, null=True)
class Meta:
managed = False
abstract = True
class TestModel(AbstractModel):
test_field = models.TextField(db_column="TEST_FIELD", blank=True, null=True)
def test_method(self):
print("just testing")
class Meta(AbstractModel.Meta):
db_table = "MY_UNMANAGED_TABLE_NAME"
# My Custom Django TestCases Implementation for my tests
def create_database(model):
with connection.schema_editor() as schema_editor:
try:
schema_editor.create_model(model)
except OperationalError:
pass
def drop_database(model):
with connection.schema_editor() as schema_editor:
try:
schema_editor.delete_model(model)
except OperationalError:
pass
class BaseModelTestCase(TransactionTestCase):
"""Custom TestCase for testing models not managed by django."""
Model = DjangoModelBase
def setUp(self):
super().setUp()
create_database(self.Model)
def tearDown(self):
super().tearDown()
drop_database(self.Model)
class AbstractBaseModelTestCase(TransactionTestCase):
"""Custom TestCase for testing abstract django models."""
Model = DjangoModelBase
def setUp(self):
# this is necessary for testing an abstract class
self.Model = DjangoModelBase(
"__TestModel__" + self.Model.__name__,
(self.Model,),
{"__module__": self.Model.__module__},
)
create_database(self.Model)
def tearDown(self):
drop_database(self.Model)
# Example of usage
class TestModelTestCase(BaseModelTestCase):
Model = TestModel
def setUp(self):
super().setUp()
self.instance = TestModel.objects.create()
self.assertIsInstance(self.instance, TestModel)
class AbstractModelTestCase(AbstractBaseModelTestCase):
Model = AbstractModel
def setUp(self):
super().setUp()
self.instance = AbstractModel.objects.create()
self.assertIsInstance(self.instance, AbstractModel)
p.s. I am not using any Django migrations as soon as my models are managed=False. So this was not necessary for me.

Simply move your unmanaged models to a dedicated app and delete migrations folder. Details in my answer in Multi db and unmanged models - Test case are failing