In my django project I am trying to migrate my database from sqlite to postgres. I've made a dump file of the database, changed my settings file to reflect the new database and now I want to upload this data to postgres. So initially I was running the following in the terminal:
python manage.py loaddata data.json
But this took very long - it was still running for more than a day, with no errors, and I eventually I stopped it.
So then I found this answer on SO, and therefore I created a load_data.py file, like so:
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings")
import django
django.setup()
from django.core.management import call_command
from django.db import transaction
DJANGO_SETTINGS_MODULE='mysite.settings'
with transaction.atomic():
call_command('loaddata', 'data.json')
And then ran it via python load_data.py. But it still takes incredibly long and the data has not loaded. My data file is 20.8MB. Is there anything that I can do to speed things up?
For reference, the model that has the most data looks like this:
class WordManager(models.Manager):
def search(self, query=None):
qs = self.get_queryset()
if query is not None:
or_lookup = (Q(target_word__icontains=query) |
Q(source_word__icontains=query)|
Q(example_sentence__icontains=query)
)
qs = qs.filter(or_lookup).distinct() # distinct() is often necessary with Q lookups
return qs
class Word(models.Model):
objects = WordManager()
target_word = models.CharField('Word in Russian', max_length=35,help_text="The word you want to add, in Russian")
source_word = models.CharField('What does it mean?',max_length=35, help_text="Write down the translation in any language", blank=False, null=True)
add_to_word_list = models.BooleanField('Would you like to create a flashcard?', default=True)
deck_name = models.CharField('Deck name', max_length=25, default="My Words")
category = models.CharField(max_length=25,blank=True,help_text="Create a custom category for this word, e.g. Food")
example_sentence = models.CharField(max_length=150,blank=True,help_text="It's important to learn words in context!")
user = models.ForeignKey(User, related_name="words",on_delete=models.CASCADE)
created_at = models.DateTimeField(auto_now=True)
def __str__(self):
return self.target_word
def get_absolute_url(self):
return reverse("vocab:detail",
kwargs={"username": self.user.username, "pk": self.pk})
class Meta:
ordering = ["target_word"]
## constrain user to add only one instance of target word
constraints = [
models.UniqueConstraint(fields=['user','target_word', 'source_word'],name='unique_word')]
from django.db.models.signals import post_save
from django.dispatch import receiver
from api.models import Flashcard, Deck
#receiver(post_save, sender=Word)
def create_deck(sender, created, instance, **kwargs):
if created:
instance._meta.model.objects.all()
deck_name = instance.deck_name
user = instance.user
flash_wanted = Word.objects.filter(add_to_word_list=True)
if flash_wanted:
deck_name, created = Deck.objects.get_or_create(owner=user, name=deck_name)
post_save.connect(create_deck, sender=Word)
#receiver(post_save, sender=Word)
def flash_from_word(sender, created, instance, **kwargs):
if created:
flash_wanted = Word.objects.filter(add_to_word_list=True)
instance._meta.model.objects.all()
target_word = instance.target_word
source_word = instance.source_word
deck_name = instance.deck_name
user = instance.user
if flash_wanted:
flash = Flashcard.objects.create(owner=user, question=target_word,answer=source_word,deck=Deck.objects.get(name=deck_name,owner=user))
post_save.connect(flash_from_word, sender=Word)
Related
class Trade(models.Model):
pips = models.FloatField(default=0)
direction = models.CharField(max_length=30)
new_balance = FloatField(default=0.0)
...
class Summary(models.Model):
winning_trades = models.IntegerField(default=0)
account_balance = FloatField(default=0.0)
...
When a user post a request he/she will populate the Trade model, this will update the summary model and send back to the user new summary data. How can I do this in an elegant way?
You're most likely looking for Django Signals. You'd want your Trade model's create event to trigger a post_save signal that a listener will receive and process.
Assuming you have saved your models in a file models.py,
Create a file signals.py with the following:
# code
from django.db.models.signals import post_save
from django.dispatch import receiver
from .models import Trade, Summary
#receiver(post_save, sender=Trade)
def update_summary(sender, instance, created, **kwargs):
if created:
# query to update Summary as needed
You'll have to add the signals to your app config.
in your apps.py of the relevant app, add the following:
from django.apps import AppConfig
class AppnameConfig(AppConfig):
name = 'appname'
**def ready(self):
import appname.signals**
First, I will encourage you to create a transaction to perform these two actions. If the second one fails, your database will remain consistent.
Then, Django allows you to override the model methods such as save. You should try that with something like the following:
class Trade():
...
def save(self, *args, **kwargs):
with transaction.atomic():
super.save()
update_summary()
Then, in the view, you could query for the recently updated Summary and return it.
class TradeViewSet():
def create(self, request, *args, **kwargs):
user = request.user
trade = Trade.create(request.data)
updated_summary = get_summary(user.id)
response = SummarySerializer(updated_summary)
return Response(response)
How to write tests for models.py file?
from django.db import models
from django.conf import settings
from django.utils import timezone
from django.contrib.auth.models import User
class Profile(models.Model): #new
user = models.OneToOneField(User, on_delete=models.CASCADE)
institute = models.CharField(max_length=200, null=True)
department = models.CharField(max_length=200, null=True)
def __str__(self):
return 'Email:{} Department:{} Institute:{}'.format(self.user.email, self.institute, self.department)
class Access(models.Model):
user = models.ManyToManyField(User)
endpoint = models.CharField(
max_length=20,
choices=(
('pubmed', 'pubmed'),
('foster', 'foster'),
('med_apps', 'med_apps'),
('mayerson_transcripts', 'mayerson_transcripts'),
('all', 'all')
),
unique=True
)
def __str__(self):
return self.endpoint
class Request(models.Model):
user = models.ForeignKey(User, on_delete=models.CASCADE, null=True)
created_time = models.DateTimeField(default=timezone.now)
url = models.CharField(max_length=1000)
def start(self):
self.created_time = timezone.now()
self.save()
def __str__(self):
return 'url:{} time:{}'.format(self.url, self.created_time)
Need to run the test in Visual Studio Code.
Also, I have seen a lot of examples for writing tests, but again got confused. It seems like I need to work on basics.
depend on where is your logic (in django views or in models) and where actions took place, you write your tests, in example above I don't see any logic or action, I'll give you simple example, below is an example to write test on a sign up view:
from django.test import Client
from django.test import TestCase
from account.models import Account
class TestSequenceFunctions(TestCase):
def test_signup_index_view(self): #Test number 1
response = self.client.post(reverse('acc:signup'), data=
{'fname':'ftest',
'lname':'ltest',
'email':'foo2#bar.com',
'password':'1111',
'pass2':'1111',
'field':'something',
'degree':'1',
}, follow=True)
self.assertEqual(response.status_code, 200)
Create a test_models.py file - note that it is recommended to have a separate test file for views, models, apis, etc:
test_views.py
test_api.py
test_models.py
test_tasks.py
Your models are quite simple and there is nothing much that you can test. You can for example test your Request model and if the start function is changing the created_time field. For example:
def test_start(mocker):
request_model = RequestFactory.create()
request_model.start()
assert request_model.created_time == timezone.now()
You can also mock the "save" method and assert that it is being called. For example:
save_patch = mocker.patch('app_name.models.Request.save')
save_patch.assert_called_once()
save_patch.assert_called_once_with(params)
I would recommend starting with simpler example, not timezone field as I'm not sure if there is a delay the test might not pass.
You can find how to mock objects and assert that they are called
Use Factory Boy and DjangoModelFactory to create fake objects for testing (such as the RequestFactory in the example).
I'm trying to develop a website for an online store, and after creating and registering models, I don't know why, this error is thrown. What can I do? And also after running the migrate command, it is saying no migrations to apply. How can I do this?
My models.py:
from django.db import models
import stripe
from django.conf import settings
from django.contrib.auth.signals import user_logged_in
from django.contrib.auth.models import User
stripe.api_key = settings.STRIPE_SECRET_KEY
class UserStripe(models.Model):
user = models.OneToOneField(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
stripe_id = models.CharField(max_length=120)
def __unicode__(self):
return str(self.stripe_id)
def get_or_create_stripe(sender, user, *args, **kwargs):
try:
user.userstripe.stripe_id
except UserStripe.DoesNotExist:
customer = stripe.Customer.create(
email = str(user.email)
)
new_user_stripe = UserStripe.objects.create(
user = user,
stripe_id = customer.id
)
except:
pass
user_logged_in.connect(get_or_create_stripe)
Have you changed anything since your last migration ?
I think you might have changed some models, now django is unable to find your table.
I would delete the current migration files and create new ones.
I have a problem where deleting an object form the admin won't delete the file associated with it. after some research I decided to implement a post_delete in the model.
For some reason I am not able to make the s3 delete the file, even after searching numerous guides and snippets, maybe someone here knows.
I use django 1.5 and boto.
Heres my code for the model:
from django.db import models
from django.contrib.auth.models import User
from fileservice.formatChecker import ContentTypeRestrictedFileField
from south.modelsinspector import add_introspection_rules
import os
from django.dispatch import receiver
from django.utils.translation import ugettext_lazy as _
from django.core.files.storage import default_storage as storage
add_introspection_rules([
(
[ContentTypeRestrictedFileField], # Class(es) these apply to
[], # Positional arguments (not used)
{ # Keyword argument
"content_types": ["content_types", {}],
"max_upload_size": ["max_upload_size", {}]
},
),
], ["^fileservice\.formatChecker\.ContentTypeRestrictedFileField"])
class Contentfile(models.Model):
content = ContentTypeRestrictedFileField(upload_to='uploads/', content_types=['video/mp4', 'application/pdf', 'image/gif', 'image/jpeg', 'image/png'],max_upload_size=5242880,blank=True, null=True, help_text='Upload a file to add it to the content the app displayes')
created_at = models.DateTimeField(auto_now_add=True, editable=False)
updated_at = models.DateTimeField(auto_now=True, editable=False)
title = models.CharField(max_length=255, unique=True)
file_type = models.CharField(max_length=5)
published = models.BooleanField(default=True)
file_owner = models.ForeignKey(User, related_name='Contentfiles')
class Meta:
ordering = ["title"]
def __unicode__(self):
return self.title
def save(self, *args, **kwargs):
file_name = os.path.basename(self.content.name)
self.file_type = file_name.split('.')[-1]
self.title = file_name.split('.')[0]
self.published = True
super(Contentfile, self).save(*args, **kwargs)
#receiver(models.signals.post_delete, sender=Contentfile)
def auto_delete_file_on_delete(sender, instance, **kwargs):
"""Deletes file from filesystem
when corresponding `MediaFile` object is deleted.
"""
if instance.content:
if os.path.isfile(storage.open(instance.content.path)):
os.remove(storage.open(instance.content.path))
#receiver(models.signals.pre_save, sender=Contentfile)
def auto_delete_file_on_change(sender, instance, **kwargs):
"""Deletes file from filesystem
when corresponding `MediaFile` object is changed.
"""
if not instance.pk:
return False
try:
old_file = Contentfile.objects.get(pk=instance.pk).content
except Conentfile.DoesNotExist:
return False
new_file = instance.content
if not old_file == new_file:
if os.path.isfile(storage.open(old_file.path)):
os.remove(storage.open(old_file.path))
It is MUCH safer to do post_delete. If something goes wrong you will start missing S3 files and you wont notice it because your DB records are intact. post_delete will be safer since it is less likely that S3 delete operation would fail after you have deleted your db record. Furthermore even if file delete fails you will be left with a bunch of unreferenced S3 file which are harmless and can be easily cleaned up.
#receiver(models.signals.post_delete, sender=Picture)
def remove_file_from_s3(sender, instance, using, **kwargs):
instance.img.delete(save=False)
You need to call FieldFile's delete() method to remove the file in S3. In your case, add a pre_delete signal where you call it:
#receiver(models.signals.pre_delete, sender=ContentFile)
def remove_file_from_s3(sender, instance, using):
instance.content.delete(save=False)
Try django-cleanup, it automatically invokes delete method on FileField when you remove model.
This worked for me by deleting files both in DB and in AWS S3.
from django.db import models
from django.dispatch import receiver
from django.views import generic
from project.models import ContentFile
from django.contrib.auth.mixins import LoginRequiredMixin,UserPassesTestMixin
class DeleteFileView(LoginRequiredMixin,UserPassesTestMixin,generic.DeleteView):
model = ContentFile
template_name = 'file-delete.html'
success_url = 'redirect-to'
#Check if the owner of the file is the one trying to delete a file
def test_func(self):
obj = self.get_object()
if obj.user == self.request.user:
return True
return False
#This code does magic for S3 file deletion
#receiver(models.signals.pre_delete, sender=ContentFile)
def remove_file_from_s3(sender, instance, using, **kwargs):
instance.image_file.delete(save=False)
I am using pre_delete you can check the django documentation.File reference deletion in DB is done by DeleteView, I hope this helps someone
I am new to django testing and have some issues using them to test relationship between models.
Here is an extract of my models:
class Member(models.Model):
user = models.OneToOneField('auth.User')
points = models.IntegerField()
def __unicode__(self):
return self.points
def get_number_of_poll(self):
nbr_of_poll = Poll.objects.filter(user=self.user).count()
return nbr_of_poll
class Poll(models.Model):
question = models.CharField(max_length=200)
user = models.ForeignKey(User)
def __unicode__(self):
return self.question
And here are the tests:
from polls.models import Member, Poll
from django.contrib.auth.models import User
from django.test import TestCase
class MemberTestCase(TestCase):
def setUp(self):
user = User(username='user')
self.member = Member(user=user, points=5000)
poll = Poll(question='poll', user=user)
def test_get_number_of_poll(self):
self.assertEqual(self.member.get_number_of_poll(), 1)
The issue is with the test_get_number_of_poll() method that always returns 0. The code works as expected on the site.
Am I doing something wrong with the test? I am not sure how I am supposed to set the poll in the test class.
You don't save any of the items you create in your setUp method. Instantiating a model object doesn't save it to the database: you should either call save() on them, or just use User.objects.create(username='user') etc which does the save for you.
The problem is that
poll = Poll(question='poll', user=user)
Only instantiates the Poll object, use the manager to actually save the object, e.g.
poll = Poll.objects.create(question='poll', user=user)