Django Test "database table is locked" - python

In my Django project, I have a view that when a user posts a zip file, it will respond immediately back and then process the data in the background with the help of threading. The view works fine in normal test, but when I run the Django's test it fails with a database table is locked error. Currently, I'm using default SQLite database and I know if I switch to another database this problem may be solved but I'm seeking an answer for the current setup. I trimmed the code for simplicity.
It seems that the problem is with writing in DeviceReportModel table. But I'm not sure why TestDeviceReport accessing it.
Model.py:
class DeviceReportModel(models.Model):
device_id = models.PositiveIntegerField(primary_key=True)
ip = models.GenericIPAddressField()
created_time = models.DateTimeField(default=timezone.now)
report_file = models.FileField(upload_to="DeviceReport")
device_datas = models.ManyToManyField(DeviceDataReportModel)
def __str__(self):
return str(self.id)
Serializers.py:
class DeviceReportSerializer(serializers.ModelSerializer):
class Meta:
model = DeviceReportModel
fields = '__all__'
read_only_fields = ('created_time', 'ip', 'device_datas')
views.py:
from django.utils import timezone
from django.core.files.base import ContentFile
from rest_framework.response import Response
from rest_framework import status, generics
import time
import threading
from queue import Queue
class DeviceReportHandler:
ReportQueue = Queue()
#staticmethod
def save_datas(device_object, request_ip, b64datas):
device_data_models = []
# ...
# process device_data_models
# this will take some time
time.sleep(10)
return device_data_models
#classmethod
def Check(cls):
while(True):
if not cls.ReportQueue.empty():
report = cls.ReportQueue.get()
# ...
report_model = DeviceReportModel(
device_id=report['device_object'], ip=report['request_ip'])
# THIS LINE GIVES ERROR
report_model.report_file.save(
"Report_{}.txt.gz".format(timezone.now()), ContentFile(report['report_data']))
device_data_models = cls.save_datas(
report['device_object'], report['request_ip'], 'SomeData')
report_model.device_datas.set(device_data_models)
report_model.save()
print("Report Handle Done")
time.sleep(.1)
#classmethod
def run(cls):
thr = threading.Thread(target=cls.Check)
thr.daemon = True
thr.start()
class DeviceReportView(generics.ListCreateAPIView):
queryset = DeviceReportModel.objects.all()
serializer_class = DeviceReportSerializer
DeviceReportHandler.run()
def post(self, request):
# ...
report = {
'device_object': 1,
'request_ip': '0.0.0.0',
'report_data': b'Some report plain data',
}
# add request to ReportQueue
DeviceReportHandler.ReportQueue.put(report)
return Response("OK", status.HTTP_201_CREATED)
tests.py:
from rest_framework.test import APITestCase
import gzip
from io import BytesIO
import base64
import time
class TestDeviceReport(APITestCase):
#classmethod
def setUpTestData(cls):
# add a new test device for other tests
pass
def generate_device_data(self):
# generate fake device data
return ""
def test_Report(self):
# generate device data
device_data = ''
for i in range(10):
device_data += self.generate_device_data() + '\n'
buf = BytesIO()
compressed = gzip.GzipFile(fileobj=buf, mode="wb")
compressed.write(device_data.encode())
compressed.close()
b64data = base64.b64encode(buf.getvalue()).decode()
data = {
"device_id": 1,
"report_data": b64data
}
response = self.client.post(
'/device/reports/', data=data, format='json')
print(response.status_code, response.content)
def tearDown(self):
# put some sleep to check whether the data has been processed
# see "Report Handle Done"
time.sleep(10)
And here is error log:
(myDjangoEnv) python manage.py test deviceApp.tests.tests.TestDeviceReport
Creating test database for alias 'default'...
System check identified no issues (0 silenced).
201 b'"OK"'
Exception in thread Thread-1:
Traceback (most recent call last):
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\site-packages\django\db\backends\sqlite3\base.py", line 383, in execute
return Database.Cursor.execute(self, query, params)
sqlite3.OperationalError: database table is locked
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\threading.py", line 917, in _bootstrap_inner
self.run()
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "<project_path>\deviceApp\views.py", line 303, in Check
"Report_{}.txt.gz".format(timezone.now()), ContentFile(report['report_data']))
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\site-packages\django\db\models\fields\files.py", line 93, in save
self.instance.save()
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\site-packages\django\db\models\base.py", line 741, in save
force_update=force_update, update_fields=update_fields)
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\site-packages\django\db\models\base.py", line 779, in save_base
force_update, using, update_fields,
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\site-packages\django\db\models\base.py", line 870, in _save_table
result = self._do_insert(cls._base_manager, using, fields, update_pk, raw)
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\site-packages\django\db\models\base.py", line 908, in _do_insert
using=using, raw=raw)
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\site-packages\django\db\models\manager.py", line 82, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\site-packages\django\db\models\query.py", line 1186, in _insert
return query.get_compiler(using=using).execute_sql(return_id)
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\site-packages\django\db\models\sql\compiler.py", line 1335, in execute_sql
cursor.execute(sql, params)
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\site-packages\django\db\backends\utils.py", line 67, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\site-packages\django\db\backends\utils.py", line 76, in _execute_with_wrappers
return executor(sql, params, many, context)
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\site-packages\django\db\utils.py", line 89, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\Masoud\Anaconda3\envs\myDjangoEnv\lib\site-packages\django\db\backends\sqlite3\base.py", line 383, in execute
return Database.Cursor.execute(self, query, params)
django.db.utils.OperationalError: database table is locked
.
----------------------------------------------------------------------
Ran 1 test in 10.023s
OK
Destroying test database for alias 'default'...

Database is locked errors
SQLite is meant to be a lightweight database, and thus can’t support a high level of concurrency. OperationalError: database is locked errors indicate that your application is experiencing more concurrency than sqlite can handle in default configuration. This error means that one thread or process has an exclusive lock on the database connection and another thread timed out waiting for the lock the be released.
Python’s SQLite wrapper has a default timeout value that determines how long the second thread is allowed to wait on the lock before it times out and raises the OperationalError: database is locked error.
If you’re getting this error, you can solve it by:
Switching to another database backend. At a certain point SQLite becomes too “lite” for real-world applications, and these sorts of concurrency errors indicate you’ve reached that point.
Rewriting your code to reduce concurrency and ensure that database transactions are short-lived.
Increase the default timeout value by setting the timeout database option:
'OPTIONS': {
# ...
'timeout': 20,
# ...
}
This will make SQLite wait a bit longer before throwing “database is locked” errors; it won’t really do anything to solve them.
https://docs.djangoproject.com/en/3.0/ref/databases/#database-is-locked-errorsoption

Try using django.test.TransactionTestCase instead of TestCase

Related

Django: Overriding Postgres Database wrapper to set search path throws error

I want to establish a connection with a postgres database in django. As it doesn't provide support for postgres schemas I am trying to set the search_path immediately after establishing a connection. To achieve this I have subclassed the DatabaseWrapper class and overridden the _cursor method as below:
from django.db.backends.postgresql.base import DatabaseWrapper
class DatabaseWrapper(DatabaseWrapper):
def __init__(self, *args, **kwargs):
super(DatabaseWrapper, self).__init__(*args, **kwargs)
def _cursor(self, name=None):
cursor = super(DatabaseWrapper, self)._cursor(name)
cursor.execute('SET search_path = schema_name')
return cursor
Now the above code works fine for the application code that we have written but when I try access the detail screen of any object in the admin panel of django, I get the below error trace:
File "/Users/azharuddin.syed/Desktop/application/custom_db_engine/base.py", line 13, in \_cursor
cursor.execute('SET search_path = schema_name')
File "/Users/azharuddin.syed/Desktop/application/venv/lib/python3.9/site-packages/django/db/backends/utils.py", line 98, in execute
return super().execute(sql, params)
File "/Users/azharuddin.syed/Desktop/application/venv/lib/python3.9/site-packages/django/db/backends/utils.py", line 66, in execute
return self.\_execute_with_wrappers(sql, params, many=False, executor=self.\_execute)
File "/Users/azharuddin.syed/Desktop/application/venv/lib/python3.9/site-packages/django/db/backends/utils.py", line 75, in \_execute_with_wrappers
return executor(sql, params, many, context)
File "/Users/azharuddin.syed/Desktop/application/venv/lib/python3.9/site-packages/django/db/backends/utils.py", line 84, in \_execute
return self.cursor.execute(sql, params)
File "/Users/azharuddin.syed/Desktop/application/venv/lib/python3.9/site-packages/django/db/utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/Users/azharuddin.syed/Desktop/application/venv/lib/python3.9/site-packages/django/db/backends/utils.py", line 82, in \_execute
return self.cursor.execute(sql)
django.db.utils.ProgrammingError: syntax error at or near "SET"
LINE 1: ...6171701248_sync_1" NO SCROLL CURSOR WITH HOLD FOR SET search...
If I understand correctly then two queries are getting mixed together and getting executed at the same time. Why is this the case?
P.S: I am aware that we can pass options parameter with the search path while defining the connections in django but this is not working in my case as the DB is behind a proxy. Any other solutions will be welcomed.

Django Celery IntegrityError

I want to create a progress bar for my project. I have a class and this class has some functions. Especially, one of them takes a long time (def download_all) and this is my main reason for wanting to create a progress bar.
I successfully set up celery, celery-progress, etc. and they work all fine. My problem is this: I want to integrate the progress bar to download_all function. I
It gives an error: IntegrityError at /o.../k...
NOT NULL constraint failed: django_celery_results_taskresult.task_id
How can I solve it?
functions.py
class myClass():
def __init__(self, n_user, n_password, n_url, n_port, db_password, username):
...
self.download_all(n_user, n_password, n_url, n_port, db_password)
...
#shared_task(bind=True, name="my_add")
def download_all(n_user, n_password, n_url, n_port, db_password)
...
len_scans = len(scans)
progress_recorder = ProgressRecorder(self)
for s in scans:
i = 0
progress_recorder.set_progress(i + 1, len_scans)
i += 1
views.py
def setup_wizard(request):
...
functions.Zafiyet(setup.n_username, setup.n_password,
setup.n_url, setup.n_port, setup.db_password,
username=request.user.username)
traceback
Environment:
Request Method: POST
Request URL: http://127.0.0.1:8000/operasyonmerkezi/konfigurasyon
Django Version: 3.2.7
Python Version: 3.9.6
Installed Applications:
['django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'dashboard',
'accounts',
'logs',
'crispy_forms',
'django_apscheduler',
'easy_timezones',
'django_celery_results',
'celery_progress']
Installed Middleware:
['django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware']
Traceback (most recent call last):
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\query.py", line 581, in get_or_create
return self.get(**kwargs), False
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\query.py", line 435, in get
raise self.model.DoesNotExist(
During handling of the above exception (TaskResult matching query does not exist.), another exception occurred:
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\backends\sqlite3\base.py", line 423, in execute
return Database.Cursor.execute(self, query, params)
The above exception (NOT NULL constraint failed: django_celery_results_taskresult.task_id) was the direct cause of the following exception:
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\core\handlers\exception.py", line 47, in inner
response = get_response(request)
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\core\handlers\base.py", line 181, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\contrib\auth\decorators.py", line 21, in _wrapped_view
return view_func(request, *args, **kwargs)
File "C:\Users\edeni\Desktop\hawkdragon\dashboard\views.py", line 128, in setup_wizard
task = (functions.myClass(setup.n_username, setup.n_password,
File "C:\Users\edeni\Desktop\hawkdragon\dashboard\functions.py", line 44, in __init__
self.download_all(n_user, n_password, n_url, n_port, db_password)
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\celery\local.py", line 188, in __call__
return self._get_current_object()(*a, **kw)
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\celery\app\task.py", line 389, in __call__
return self.run(*args, **kwargs)
File "C:\Users\edeni\Desktop\hawkdragon\dashboard\functions.py", line 153, in download_all
progress_recorder.set_progress(i + 1, len_scans)
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\celery_progress\backend.py", line 46, in set_progress
self.task.update_state(
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\celery\app\task.py", line 971, in update_state
self.backend.store_result(
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\celery\backends\base.py", line 482, in store_result
self._store_result(task_id, result, state, traceback,
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django_celery_results\backends\database.py", line 66, in _store_result
self.TaskModel._default_manager.store_result(
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django_celery_results\managers.py", line 46, in _inner
return fun(*args, **kwargs)
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django_celery_results\managers.py", line 168, in store_result
obj, created = self.using(using).get_or_create(task_id=task_id,
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\query.py", line 588, in get_or_create
return self.create(**params), True
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\query.py", line 453, in create
obj.save(force_insert=True, using=self.db)
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\base.py", line 726, in save
self.save_base(using=using, force_insert=force_insert,
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\base.py", line 763, in save_base
updated = self._save_table(
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\base.py", line 868, in _save_table
results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\base.py", line 906, in _do_insert
return manager._insert(
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\query.py", line 1270, in _insert
return query.get_compiler(using=using).execute_sql(returning_fields)
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\sql\compiler.py", line 1416, in execute_sql
cursor.execute(sql, params)
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\backends\utils.py", line 98, in execute
return super().execute(sql, params)
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\backends\utils.py", line 66, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\backends\utils.py", line 75, in _execute_with_wrappers
return executor(sql, params, many, context)
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\backends\sqlite3\base.py", line 423, in execute
return Database.Cursor.execute(self, query, params)
Exception Type: IntegrityError at /operasyonmerkezi/konfigurasyon
Exception Value: NOT NULL constraint failed: django_celery_results_taskresult.task_id
If you want to check progress of task you have 3 options
store current progress as model field in db
store as key value in redis db
count at runtime by filtering and count element in db with changed status
Frontend can get this result by common REST API or websocket.
example minimum configs for REST API
model.py
class MyLongProcess(models.Model):
active_uuid = models.UUIDField('Active process', null=True, blank=True)
name = models.CharField('Name', max_length=255)
current_step = models.IntegerField('Current step', default=0)
total = models.IntegerField('Total', default=0)
#property
def percentage_sending(self):
# or it can be computed by filtering elements processed in celery with complete status
return int((current_step / total) * 100)
views.py
def initiate_execute_long_task(request, name):
process = MyLongProcess.objects.create(active_uuid=uuid.uuid4(), name=name, total=BIG_COUNTED_VALUE)
async_execute_long_task.delay(process)
return HttpResponse()
from app.celery import app
#app.task
def async_execute_long_task(process)
for i in range process.total:
do_some_staff()
process.current_step += 1
process.save()
detail.html
<div class="progress-bar" role="progressbar"
style="width: {{ my_model_object.percentage_sending }}%;"
aria-valuenow="{{ my_model_object.percentage_sending }}"
aria-valuemin="0" aria-valuemax="100">{{ my_model_object.percentage_sending }}%
</div>
do not use self.download_all(n_user, n_password, n_url, n_port, db_password)
but use it self.download_all.delay(n_user, n_password, n_url, n_port, db_password) to run function in celery task
you can use websocket and send data for progress to html and update progress bar with javascript
each time progress updated progress bar will move
First, you probably do not want to use the #shared_task decorator on a method like that.
Your method signature is kind of mangled because of this. The bind=True parameter will cause celery to pass the task object into the call, which may produce unexpected results based on your signature.
Demonstration of what I mean by this:
#shared_task(bind=True, name='mytask')
def my_task(the_task, a, b):
print(the_task) # a celery Task object
return a + b
# OR
#shared_task(name='mytask')
def my_task(a, b):
return a + b
Note that these are ordinary functions, not methods in a class.
Now, compound this with the fact that methods on classes implicitly receive self, the instance of the class, as the first argument. This is also missing from your signature. It's honestly surprising it executes at all as-written... I would have expected it to fail with a TypeError based on the code you provided.
Instead, you should factor out the task to its own function or add the #staticmethod decorator over the method first:
class Foo:
#shared_task(name='mytask')
#staticmethod
def bar(a, b):
...
Winding back your stacktrace, your error comes from an integrity error in the task result table:
Exception Value: NOT NULL constraint failed: django_celery_results_taskresult.task_id
This happens because your app is, for one reason or another, trying to re-use a task ID and trying to store the result that has already been executed and stored in the DB. This is potentially caused by the issues described above, a bad concurrency model in your app, or an out-of-band change to the database/queue that caused unexpected duplication. You need to either flush the offending task out of your message queue or remove the offending result from the database, but there may not be guarantees this won't happen again unless you fix the underlying problem with your application code.
This seems like came from Django model: "self.model.DoesNotExist".
It is not come from celery, from Model.objects.
You'd better check your ORMs.

Django Test - South migration reports 'no such table' but I can see said table in the db

I'm working with an implementation of Django using py2exe & Qt. There is a SQLite db which is managed using South.
Running django tests on the source is fine with a memory database but I'm also trying to implement testing using PhantomCSS (PhantomJS) to perform CSS regression testing.
To do this I have a subclass of LiveServerTestCase (source). I'm running Django tests using a sqlite database on disk, loading in fixtures through a call to loaddata when starting a custom server process (see server function at the end).
The tests for it looks like this;
class PhantomTestBackupRestore(PhantomTestCase):
fixtures = ['basic_db.json',]
def test_backup(self):
self.assertTrue(
self.phantom(RUNNER_PATH,
screenID='lupyvAQL',
host='http://127.0.0.1',
port=buildconstants.PRODUCT_LISTENPORT)
)
The fixture which I'm creating with the following command (this loaded yesterday so seems to be a temperamental issue with loaddata);
manage.py dumpdata -n --indent 4 --exclude=contenttypes --exclude=auth --format=json > phantom/fixtures/basic_db.json
I'm getting the following stacktrace;
[exec] Traceback (most recent call last):
[exec] File "C:\Python27\lib\multiprocessing\process.py", line 258, in _bootstrap
[exec] self.run()
[exec] File "C:\Python27\lib\multiprocessing\process.py", line 114, in run
[exec] self._target(*self._args, **self._kwargs)
[exec] File "C:\Users\markw\work\bptrti3b\src\django_offline\startuphelpers.py", line 124, in django_server_helper
[exec] *fixture_list
[exec] File "C:\Users\markw\work\bptrti3b\src\django\core\management\base.py", line 283, in execute
[exec] output = self.handle(*args, **options)
[exec] File "C:\Users\markw\work\bptrti3b\src\django\core\management\commands\loaddata.py", line 55, in handle
[exec] self.loaddata(fixture_labels)
[exec] File "C:\Users\markw\work\bptrti3b\src\django\core\management\commands\loaddata.py", line 84, in loaddata
[exec] self.load_label(fixture_label)
[exec] File "C:\Users\markw\work\bptrti3b\src\django\core\management\commands\loaddata.py", line 140, in load_label
[exec] obj.save(using=self.using)
[exec] File "C:\Users\markw\work\bptrti3b\src\django\core\serializers\base.py", line 164, in save
[exec] models.Model.save_base(self.object, using=using, raw=True)
[exec] File "C:\Users\markw\work\bptrti3b\src\django\db\models\base.py", line 578, in save_base
[exec] updated = self._save_table(raw, cls, force_insert, force_update, using, update_fields)
[exec] File "C:\Users\markw\work\bptrti3b\src\django\db\models\base.py", line 638, in _save_table
[exec] updated = self._do_update(base_qs, using, pk_val, values, update_fields)
[exec] File "C:\Users\markw\work\bptrti3b\src\django\db\models\base.py", line 676, in _do_update
[exec] return base_qs.filter(pk=pk_val)._update(values) > 0
[exec] File "C:\Users\markw\work\bptrti3b\src\django\db\models\query.py", line 509, in _update
[exec] return query.get_compiler(self.db).execute_sql(None)
[exec] File "C:\Users\markw\work\bptrti3b\src\django\db\models\sql\compiler.py", line 971, in execute_sql
[exec] cursor = super(SQLUpdateCompiler, self).execute_sql(result_type)
[exec] File "C:\Users\markw\work\bptrti3b\src\django\db\models\sql\compiler.py", line 777, in execute_sql
[exec] cursor.execute(sql, params)
[exec] File "C:\Users\markw\work\bptrti3b\src\django\db\utils.py", line 105, in inner
[exec] return func(*args, **kwargs)
[exec] File "C:\Users\markw\work\bptrti3b\src\django\db\utils.py", line 99, in __exit__
[exec] six.reraise(dj_exc_type, dj_exc_value, traceback)
[exec] File "C:\Users\markw\work\bptrti3b\src\django\db\utils.py", line 105, in inner
[exec] return func(*args, **kwargs)
[exec] File "C:\Users\markw\work\bptrti3b\src\django\db\backends\sqlite3\base.py", line 445, in execute
[exec] return Database.Cursor.execute(self, query, params)
[exec] OperationalError: Problem installing fixture 'C:\Users\markw\work\src\phantom\fixtures\basic_db.json':
Could not load sites.Site(pk=1): no such table: django_site
Because I'm dealing with multiprocessing, I'm not sure if manage.py is perhaps using a memory database or something when these errors occur, but with the following database setup I didn't think it used memory for tests;
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(
APPDATA_DIR, 'sqlite3.db',
),
'TEST_NAME': os.path.join(
APPDATA_DIR, 'test.db',
)
}
}
The function for starting the server if it's helpful;
def django_server_helper(qt_pipe=None, fixture_list=None):
"""
This is for use in testing-only modes (e.g. PhantomCSS).
If this process is a subprocess of the main application, qt_pipe will be
supplied to allow us to access functionality provided by django_offline.
The other end of this pipe comes out in the DjangoOfflineApp instance.
#type qt_pipe: multiprocessing.Connection
#param qt_pipe: Pipe for communicating with the main django_offline app.
#type fixture_list: list or None
#param fixture_list: List of JSON fixture files to load in for testing.
#rtype: None
"""
use_threading = True
if fixture_list is not None:
# A fixture list has been supplied, so we're in test mode.
from django.core.management.commands import loaddata
loaddata.Command().execute(
verbosity = 0,
database = "default",
settings = "mysite.settings",
*fixture_list
)
if qt_pipe is not None:
# A pipe has been supplied, so we're a subprocess
from django_offline import connector
connector.QT_PIPE = qt_pipe
# start the django server up
from django_offline import settings_central
try:
from django.contrib.staticfiles.management.commands import runserver
runserver.Command().execute(
use_threading=use_threading,
use_static_handler=True,
insecure_serving=True,
addrport='127.0.0.1:{0}'.format(settings_central.LISTEN_PORT),
)
except socket.error as e:
logging.exception("Socket occupied; not starting a server.")
sys.exit(1)
sys.exit(0)
Do I need to call syncdb before running loaddata or is this something that can be done with a migration call to South with the fixture list?
The problem is that the code is running loaddata before the database table has been set up, hence it can't write the Site value.
Normally manage.py test takes care of setting up the test databases - are you running using a different / custom test runner? If not, then try stepping through manage.py test to see what the initialisation order of your test code versus the calls to syncdb are.
If you're using your own test runner, then add appropriate calls to syncdb. Make sure you pick up the South version if using South, and/or set SOUTH_TESTS_MIGRATE = False

django.db.utils.DatabaseError

I'm setting up a django model to store regions, like USA, Germany, etc. I made the region name unique for the table. I have a script that populates the database from a list and if there is a duplicate region name IntegrityError is thrown as expected but then another error happens and I can't tell why from the error message. Any ideas? Thanks!
django.db.utils.DatabaseError: current transaction is aborted, commands ignored until end of transaction block
Model:
class Region(models.Model):
name = models.CharField(max_length=512, unique=True)
def __unicode__(self):
return self.name
Populate code:
try:
Region(name=server['locale']).save()
print 'Added region: %(locale)s' % server
except IntegrityError:
pass
I've confirmed that the IntegrityError is occuring but then I get this error which I dont expect:
File "/home/daedalus/webapps/wowstatus/lib/python2.6/django/db/models/base.py", line 456, in save
self.save_base(using=using, force_insert=force_insert, force_update=force_update)
File "/home/daedalus/webapps/wowstatus/lib/python2.6/django/db/models/base.py", line 549, in save_base
result = manager._insert(values, return_id=update_pk, using=using)
File "/home/daedalus/webapps/wowstatus/lib/python2.6/django/db/models/manager.py", line 195, in _insert
return insert_query(self.model, values, **kwargs)
File "/home/daedalus/webapps/wowstatus/lib/python2.6/django/db/models/query.py", line 1518, in insert_query
return query.get_compiler(using=using).execute_sql(return_id)
File "/home/daedalus/webapps/wowstatus/lib/python2.6/django/db/models/sql/compiler.py", line 788, in execute_sql
cursor = super(SQLInsertCompiler, self).execute_sql(None)
File "/home/daedalus/webapps/wowstatus/lib/python2.6/django/db/models/sql/compiler.py", line 732, in execute_sql
cursor.execute(sql, params)
File "/home/daedalus/webapps/wowstatus/lib/python2.6/django/db/backends/util.py", line 15, in execute
return self.cursor.execute(sql, params)
File "/home/daedalus/webapps/wowstatus/lib/python2.6/django/db/backends/postgresql_psycopg2/base.py", line 44, in execute
return self.cursor.execute(query, args)
django.db.utils.DatabaseError: current transaction is aborted, commands ignored until end of transaction block
You should reset your db state if something fails
for example:
from django.db import transaction
#transaction.commit_manually
def Populate():
try:
Region(name=server['locale']).save()
print 'Added region: %(locale)s' % server
except IntegrityError:
transaction.rollback()
else:
transaction.commit()
I get this error sometimes when accessing the database from the django shell. I fix it by closing the connection:
from django.db import connection
connection.close()
cur = connection.cursor()
sql = 'select distinct category from uploads_document'
cur.execute(sql)
after I have made an error like 'select ditsinct ..'

Django and Postgres transaction rollback

I have a piece of code that works in a background process which looks like
from django.db import transaction
try:
<some code>
transaction.commit()
except Exception, e:
print e
transaction.rollback()
In a test, I break <some_code> with data that causes a database error. The exception is following
File "/home/commando/Development/Diploma/streaminatr/stream/testcases/feeds.py", line 261, in testInterrupt
form.save(self.user1)
File "/usr/lib/pymodules/python2.5/django/db/transaction.py", line 223, in _autocommit
return func(*args, **kw)
File "/home/commando/Development/Diploma/streaminatr/stream/forms.py", line 99, in save
print(models.FeedChannel.objects.all())
File "/usr/lib/pymodules/python2.5/django/db/models/query.py", line 68, in `__repr__ `
data = list(self[:REPR_OUTPUT_SIZE + 1])
File "/usr/lib/pymodules/python2.5/django/db/models/query.py", line 83, in `__len__ `
self._result_cache.extend(list(self._iter))
File "/usr/lib/pymodules/python2.5/django/db/models/query.py", line 238, in iterator
for row in self.query.results_iter():
File "/usr/lib/pymodules/python2.5/django/db/models/sql/query.py", line 287, in results_iter
for rows in self.execute_sql(MULTI):
File "/usr/lib/pymodules/python2.5/django/db/models/sql/query.py", line 2369, in execute_sql
cursor.execute(sql, params)
InternalError: current transaction is aborted, commands ignored until end of transaction block
This is what I expect. The bad thing is that I still get the same error when I try to access the DB after transaction.rollback is called. What should I do to rollback the transaction succcessfully and make the connection usable once again?
Btw, I also tried inserting print connection.queries to debug the code, and it always returns an empty list. Could it be that Django is using some other DB connection?
The code is run outside of request-response cycle. I tried switching TransactionMiddleware on and off, but it gave no effect.
I am using Django 1.1 and Postgres 8.4.
Default TestCase does not know anything about transactions, you need to use TransactionalTestCase in this case.
I wrote this decorator based on the transaction middleware source. Hope it helps, works perfectly for me.
def djangoDBManaged(func):
def f(*args, **kwargs):
django.db.transaction.enter_transaction_management()
django.db.transaction.managed(True)
try:
rs = func(*args, **kwargs)
except Exception:
if django.db.transaction.is_dirty():
django.db.transaction.rollback()
django.db.transaction.leave_transaction_management()
raise
finally:
if django.db.transaction.is_managed():
if django.db.transaction.is_dirty():
django.db.transaction.commit()
django.db.transaction.leave_transaction_management()
return rs
# So logging gets the right call info whatever the decorator order is
f.__name__ = func.__name__
f.__doc__ = func.__doc__
f.__dict__ = func.__dict__
return f

Categories

Resources