Trying to speed up the performance of my app, and post some analysis with debug toolbar I can see I am doing 68 queries.
I query the circuits for every showroom (68 of), I thought if I just query the circuits once, could I then requery the existing query instead of calling the DB again for each Circuit?
something like adding:
crData = Circuits.objects.only('circuit_preference','site_data__id')
then how to query crData again? to match each statement?
Current code below
# get shworoom data
srData = SiteData.objects.only('location','subnet').filter(is_live=True).exclude(site_type='Major Site')
for sr in srData:
site = SiteType()
site.type = checkRoute(sr.subnet)
site.location = sr.location
if 'MPLS' in site.type:
mpls = site.type.split('-')
try:
d = Circuits.objects.only('circuit_preference','site_data').filter(site_data__id=sr.id,provider=mpls[0],circuit_type__icontains=mpls[1])
site.preference = d[0].circuit_preference
except:
pass
elif site.type == '4G':
try:
d = Circuits.objects.only('circuit_preference','site_data').filter(site_data__id=sr.id,provider='EE',circuit_type__icontains=site.type)
site.preference = d[0].circuit_preference
except:
pass
elif site.type == 'DSL' or site.type == 'FIBRE':
try:
d = Circuits.objects.only('circuit_preference','site_data').filter(site_data__id=sr.id,circuit_type__icontains=site.type)
site.preference = d[0].circuit_preference
except:
pass
**EDIT: models below
class SiteData(models.Model):
location = models.CharField(max_length=50)
site_type = models.CharField(max_length=20, verbose_name="Site Type", \
choices=settings.SITE_TYPE)
subnet = models.GenericIPAddressField(protocol='IPv4')
bgp_as = models.CharField(max_length=6, verbose_name="BGP AS Number")
opening_date = models.DateField(verbose_name="Showroom opening date")
last_hw_refresh_date = models.DateField(verbose_name="Date of latest hardware refresh", \
blank=True, null=True)
is_live = models.BooleanField(default=False, verbose_name="Is this a live site?")
tel = models.CharField(max_length=20, blank=True, null=True)
notes = models.TextField(blank=True)
class Meta:
verbose_name = "Site Data"
verbose_name_plural = "Site Data"
ordering = ('location',)
def __unicode__(self):
return self.location
class Circuits(models.Model):
site_data = models.ForeignKey(SiteData, verbose_name="Site", on_delete=models.PROTECT)
order_no = models.CharField(max_length=200, verbose_name="Order No")
install_date = models.DateField(blank=True, null=True)
circuit_type = models.CharField(max_length=100, choices=settings.CIRCUIT_CHOICES)
circuit_preference = models.CharField(max_length=20, verbose_name="Circuit Preference", \
choices=settings.CIRCUIT_PREFERENCE, blank=True, null=True)
circuit_speed_down = models.DecimalField(max_digits=10, decimal_places=1, blank=True, null=True)
circuit_speed_up = models.DecimalField(max_digits=10, decimal_places=1, blank=True, null=True)
circuit_bearer = models.IntegerField(blank=True, null=True)
provider = models.CharField(max_length=200, choices=settings.PROVIDER_CHOICES)
ref_no = models.CharField(max_length=200, verbose_name="Reference No")
cost_per_month = models.DecimalField(decimal_places=2, max_digits=8)
contract_length = models.IntegerField(verbose_name="Contact length in years")
service_contacts = models.ForeignKey(ServiceContacts, on_delete=models.PROTECT)
subnet = models.GenericIPAddressField(protocol='IPv4', verbose_name="Subnet", \
blank=True, null=True)
default_gateway = models.GenericIPAddressField(protocol='IPv4', \
verbose_name="Default Gateway", blank=True, null=True)
subnet_mask = models.CharField(max_length=4, verbose_name="Subnet Mask", \
choices=settings.SUBNET_MASK_CHOICES, blank=True, null=True)
internet_device = models.ForeignKey(ConfigTemplates, \
verbose_name="is this circuit the active internet line for a device?", \
default=6, on_delete=models.PROTECT)
decommissioned = models.BooleanField(default=False, verbose_name="Decomission this circuit?")
I was going to point you toward Pickling, but I suppose that doesn't make sense unless you need to cache the querysets to re-use in another location.
Actually I'm pretty sure querysets are pretty good for only hitting the database when they need to, which is when they're first evaluated. However, I think redeclaring the queryset would cause it to be re-evaluated, but if you create a list/dictionary of querysets I imagine you should be able to just re-use them without hitting the database again (unless you need to run new filters on them.) So I don't think you have much choice than to hit the database for each time you fetch a crData queryset, but you should at least be able to store the querysets and reuse them without it hitting the database for each one again.
Something like this should work I think. Would love to know if I'm wrong.
crData = []
for sr in srData:
# ...
crData.append(d)
for cr in crData:
# Do stuff
EDIT: Here's another relevant question: Django ORM and hitting DB
Related
I am having 4 models linked with a foreign key,
class CustomUser(AbstractUser):
username = None
email = models.EmailField(('email address'), unique=True)
phone_no = models.CharField(max_length=255, unique=True)
USERNAME_FIELD = 'email'
REQUIRED_FIELDS = []
objects = CustomUserManager()
def __str__(self):
return self.email
class personal_profile(models.Model):
custom_user = models.ForeignKey(CustomUser, on_delete=models.CASCADE)
picture = models.ImageField(default='profile_image/pro.png', upload_to='profile_image', blank=True)
role = models.CharField(max_length=255, blank=True, null=True)
gender = models.CharField(max_length=255, blank=True, null=True)
date_of_birth = models.DateField(blank=True, null=True)
def __str__(self):
return str(self.pk)
class academia_profile(models.Model):
custom_user = models.ForeignKey(CustomUser, on_delete=models.CASCADE)
education_or_certificate = models.CharField(max_length=255, blank=True, null=True)
university = models.CharField(max_length=255, blank=True, null=True)
def __str__(self):
return str(self.pk)
class contact_profile(models.Model):
custom_user = models.ForeignKey(CustomUser, on_delete=models.CASCADE)
country = models.CharField(max_length=255, blank=True, null=True)
state = models.CharField(max_length=255, blank=True, null=True)
city = models.CharField(max_length=255, blank=True, null=True)
def __str__(self):
return str(self.pk)
For extracting the data of those four models, I need to extract it by querying 4 times differently and then by passsing for different variables to HTML templates it something a hectic plus would be reducing the performance speed (I am sure!)
My current queries be like
user_base = CustomUser.objects.get(id=user_id)
user_personal = personal_profile.objects.get(custom_user=user_id)
academia = academia_profile.objects.get(custom_user=user_id)
contact = contact_profile.objects.get(custom_user=user_id)
Is it possible to get all of the four queries values in a single variable by hitting a single join query in ORM ?
also, I want to extract just the country from contact_profile and picture from personal_profile in the join query.
Select_related() can able to work here but how? that's what I am not getting.
You are looking for prefetch_related:
Returns a QuerySet that will automatically retrieve, in a single batch, related objects for each of the specified lookups.
user_base = (
CustomUser
.objects
.prefetch_related( #<-- THIS!
"personal_profile_set",
"academia_profile_set",
"contact_profile_set")
.get(id=user_id))
personal_profile = user_base.personal_profile_set.all()[0]
academia_profile = user_base.academia_profile_set.all()[0]
contact_profile = user_base.contact_profile_set.all()[0]
Btw, if you have only one personal_profile, academia_profile, contact_profile per CustomUser, consider changing ForeignKey by OneToOneField and use select_related.
The query to be implemented with ORM is as follows,
SELECT t2.*
FROM sub_menu AS t1
INNER JOIN sub_menu AS t2 ON (t1.sub_menu_id = t2.parent_sub_menu_id)
WHERE t1.sub_menu_id = 1;
The model is as follows,
class SubMenu(models.Model):
sub_menu_id = models.AutoField(primary_key=True)
menu = models.ForeignKey('commons.MainMenu', related_name='sub_menus', on_delete=models.CASCADE)
parent_sub_menu_id = models.IntegerField(blank=True, null=True)
name = models.CharField(max_length=50)
en_name = models.CharField(max_length=50, blank=True)
ord = models.IntegerField()
api = models.CharField(max_length=255, blank=True)
api_method = models.CharField(max_length=7, blank=True)
api_detail = models.CharField(max_length=255, blank=True)
menu_type_cd = models.CharField(max_length=5, blank=True)
menu_auth_type_cd = models.CharField(max_length=5)
is_common = models.BooleanField(default=False)
is_ns = models.BooleanField(default=False)
spc_auth = models.BooleanField(default=False)
spc_auth_cd = models.CharField(max_length=5, blank=True)
create_dt = models.DateTimeField(auto_now_add=True)
update_dt = models.DateTimeField(auto_now=True)
class Meta:
db_table = 'sub_menu'
unique_together = ('api', 'api_method',)
Not using a raw method, Is it possible to implement with Django's ORM?
Thank you.
You should do the relationship correctly on your model: https://docs.djangoproject.com/en/3.0/ref/models/fields/#module-django.db.models.fields.related. Then the parent_sub_menu should be:
class Submenu:
parent_sub_menu = models.ForeignKey('self', null=True, blank=True, on_delete=models.CASCADE)
Then run generate & DB migration. The query below should work.
And never declare relationship like you are doing right now, use Model instead via the documentation I sent.
Django does it for you already. You can just filter the related field.
https://docs.djangoproject.com/en/3.0/topics/db/queries/#lookups-that-span-relationships
SubMenu.objects.filter(parent_sub_menu__sub_menu_id=1)
I have the following piece of code. I'm using it to return json so Datatables can render it. I pass in query parameters.
def map_query(type_, type_model, type_pk, query_data, request):
type_results_query = None
problem_data = get_model_query_data(query_data, Problem)
problems_filtered = Problem.objects.filter(**problem_data)
if request.POST:
model_query_data = get_model_query_data(query_data, type_model)
type_results_query = Chgbk.objects.filter(**model_query_data)
print(type_results_query)
return type_results_query
So type_results_query returns data I want. But Problem model has a foreign key on it which links to key on table. I want to get the data from the Problem table into the Chgbk query as well, sort of the two objects merged but I cannot figure out how to do this and it is driving me crazy.
Models would be:
class Chgbk(VNCModel):
chgbk_id = models.IntegerField(primary_key=True)
facility = models.ForeignKey('Facility', models.DO_NOTHING)
create_dt = models.DateTimeField(blank=True, null=True)
mod_dt = models.DateTimeField(blank=True, null=True)
carrier_scac = models.CharField(max_length=25, blank=True, null=True)
carrier_name = models.CharField(max_length=25, blank=True, null=True)
class Problem(VNCModel):
problem_id = models.IntegerField(primary_key=True)
chgbk = models.ForeignKey(Chgbk, models.DO_NOTHING, blank=True, null=True)
I have a User model that has 20 count field:
class User(models.Model):
username = models.CharField(max_length=16)
password = models.CharField(max_length=40) # sha1
real_name = models.CharField(max_length=12, null=True,blank=True)
phone = models.CharField( max_length=11)
email = models.EmailField(blank=True, null=True )
qq = models.CharField(max_length=10, null=True, blank=True)
address = models.CharField(max_length=64, blank=True, null=True)
id_card = models.CharField(blank=True, null=True, max_length=18, validators=[RegexValidator(regex='^.{18}$', message='身份证长度必须为18', code='nomatch')])
id_card_img_front = models.CharField(max_length=256, blank=True, null=True)
id_card_img_back = models.CharField(max_length=256, blank=True, null=True)
nickname = models.CharField(max_length=16, blank=True, null=True)
profile = models.CharField(max_length=256, blank=True, null=True, default=' ')
usertype = models.ForeignKey(to='UserType', default=1, blank=True)
user_c_type = models.CharField(max_length=4, null=True)
fixed_phone = models.CharField(max_length=16, null=True)
fax = models.CharField(max_length=16, null=True)
main_bussiness = models.CharField(max_length=16, null=True)
main_industry = models.CharField(max_length=16, null=True)
company_name = models.CharField(max_length=32, null=True)
company_address = models.CharField(max_length=32, null=True)
province = models.CharField(max_length=32, null=True, default="--省--")
town = models.CharField(max_length=32, null=True, default="--市--") # 省市县
country_level = models.CharField(max_length=32, null=True, default="--县--")
ctime = models.DateTimeField(auto_now_add=True)
uptime = models.DateTimeField(auto_now=True)
status = models.CharField(max_length=1, null=True, default=1)
And in my Django project, I want to use Paginator to realize paging for user list.
Because the count of my user in my database is no more than 10,000 rows.
So, whether I can get all of the user in my database then paginate them?
user_all = models.User.objects.all()
paginator = Paginator(user_all, 10)
try:
user_list_page = paginator.page(page_number)
except PageNotAnInteger:
user_list_page = paginator.page(1)
except EmptyPage:
user_list_page = paginator.page(paginator.num_pages)
And I don't know whether this method(query out all the rows of user data) is inefficiency.
Or, how to weight the balance of query out the users from database? For a positive limit numbers ( over the count I should change paginate method, less than that I can use my page method )?
Or is there a better method to paginate my users?
The generic ListView brings a paginator. You should make sure that your queryset is sorted, otherwise the pages might repeat some of the objects (that is not a Django issue - paging without a sort order is just not practical).
https://docs.djangoproject.com/en/1.11/ref/class-based-views/generic-display/#listview
from django.views.generic.list import ListView
class UserListView(ListView):
template = 'my_user_list_template.html'
model = User # not django.contrib.User but your's
paginate_by = 10
def get_queryset(self):
return super().get_queryset().order_by('realname', 'pk')
The template will contain all context data required to render pagination, and the max 10 objects for the current page.
There are 3rd party modules like pure_pagination that extend the pagination to allow for other GET parameters in the pagination links: https://github.com/jamespacileo/django-pure-pagination
More links:
https://docs.djangoproject.com/en/1.11/ref/class-based-views/mixins-multiple-object/#django.views.generic.list.MultipleObjectMixin.paginate_by
See also on the same page: paginate_orphans, paginator_class etc.
I'm trying to create a one-to-one relationship matching a table with property billing records to another with physical addresses, but I keep getting this error. When I search for the error, nothing shows that is relevant to this situation.
I've no idea what "[<PropertyRecord: 242811400004>]" is referencing since it isn't a PIN number and doesn't exist in any tables.
Also, I've been through the data and there are no null values on the pin in either table.
Getting this error:
{
error: "The object '[<PropertyRecord: 242811400004>]' has an empty attribute 'pin' and doesn't allow a default or null value."
}
Models:
class PropertyRecord(models.Model):
pin = models.BigIntegerField(db_index=True, max_length=20)
date_added = models.DateField(null=True)
last_chgdte = models.DateField()
name = models.CharField(db_index=True, max_length=30, null=True)
address1 = models.CharField(max_length=100, null=True)
address2 = models.CharField(max_length=100, null=True)
city = models.CharField(max_length=50, null=True)
state = models.CharField(max_length=2, null=True)
zip = models.CharField(max_length=10, null=True)
class Meta:
unique_together = ("pin", "last_chgdte")
select_on_save = True
def __str__(self):
return str(self.pin)
class PropertyAddress(models.Model):
pin = models.BigIntegerField(db_index=True, max_length=20, unique=True)
street_address = models.CharField(max_length=50, null=True)
city_name = models.CharField(max_length=50, null=True)
zip_code = models.IntegerField(max_length=5, null=True)
class Meta:
select_on_save = True
def __str__(self):
return str(self.pin)
Resources:
class PropertyAddressResource(ModelResource):
class Meta:
queryset = PropertyAddress.objects.all()
class PropertyRecordResource(ModelResource):
full_address = fields.ToOneField(
PropertyAddressResource,
attribute=lambda bundle: PropertyRecord.objects.filter(pin=bundle.obj.pin),
full=True,
null=True
)
class Meta:
queryset = PropertyRecord.objects.all()
resource_name = 'propertyrecords'
I can't imagine that joining tables wouldn't be a common need, so it should have a simple solution.
Rather, I was able to get around the issue by using dehydrate, which feels overly complicated but works. +1 to this being a PITA with the extra step to serialize the data.
def dehydrate(self, bundle):
query_data = PropertyAddress.objects.filter(pin=bundle.obj.pin)
results = serialize('json', query_data, fields=('street_address', 'city_name', 'zip_code'))
bundle.data['full_address'] = json.loads(results)
return bundle