I'm working on import feature which allows user to create django database models from selected csv file.
Models are related which each other with foreign keys and many-to-many fields.
There is a lot
object.save()
and Object.objects.get(...) in my code which, I suppose, cause it to run so slow.
When an error (for example integrity error) occurs, I need all the changes in database to be rolled back. So I'm using
transaction.atomic
decorator on my view and it works fine.
The problem is, my import is really slow. Parsing file containing ~2000 lines (which could possibly add about 1000 objects to my database) takes about 3 minutes, which is too long.
Is there a way to make it faster? I've read about
bulk_create
function, but "It does not work with many-to-many relationships.".
If this is important, I'm using postgresql.
EDIT:
File structure looks like this:
subject_name
day [A/B] begins_at - ends_at;lecturer_info
Then multiple lines like:
student_uid;student_info
Ok, here's the code.
def csv_import(market, csv_file):
lines = [line.strip().decode('utf-8') for line in csv_file.readlines()]
lines = [line for line in lines if line]
pattern = re.compile(r'[0-9]+;.+')
week_days = {
'monday': 0,
.
.
.
}
term, subject, lecturer, student = None, None, None, None
for number, line in enumerate(lines):
if not ';' in line:
subject = Subject(subject_id=number, name=line, market=market)
subject.save()
elif not pattern.match(line):
term_info, lecturer_info = line.split(';') # term_info - 'day begins_at - ends_at', lecturer_info - lecturer
term_info = term_info.replace(' - ', ' ').split()
term = Term(term_id=number, subject=subject, day=week_days[term_info[0]], begin_at=term_info[-2],
ends_at=term_info[-1])
if len(term_info) == 4:
term.week = term_info[1]
lecturer_info = lecturer_info.rsplit(' ', 1)
try:
lecturer = Lecturer.objects.get(first_name=lecturer_info[0], last_name=lecturer_info[1])
except Lecturer.DoesNotExist:
lecturer = Lecturer(first_name=lecturer_info[0], last_name=lecturer_info[1])
lecturer.save()
term.lecturer = lecturer
term.save()
else:
gradebook_id, student_info = line.split(';')
student_info = student_info.rsplit(' ', 1)
try:
student = TMUser.objects.get(uid=int(gradebook_id))
except TMUser.DoesNotExist:
student = TMUser(uid=int(gradebook_id), username='student'+gradebook_id, first_name=student_info[0],
last_name=student_info[1], password=make_password('passwd'), user_group='user')
student.save()
student.terms.add(term)
student.save()
This is some pseudo code to show you the basic idea of what I meant by caching results:
cache = {}
for number, line in enumerate(lines):
...
elif not pattern.match(line):
...
term = Term(term_id=number, subject=subject, ...)
lecturer_id = (lecturer_info[0], lecturer_info[1]) #first name and last
if cache[lecturer_id]:
#retrieve from cache
lecturer = cache[lecturer_id]
else:
try:
lecturer = Lecturer.objects.get(first_name= lecturer_id[0], last_name= lecturer_id[1])
except Lecturer.DoesNotExist:
lecturer = Lecturer(first_name= lecturer_id[0], last_name= lecturer_id[1])
lecturer.save()
#add to cache
cache[lecturer_id] = lecturer
term.lecturer = lecturer
term.save()
#etc.
Related
I've got an assignment where I have to create a class for Employees, read in a text file containing employee information to store as an object, read in an hours worked file and the create a payment information dictionary out of the files. So, far it seems to be working fine as long as their is only one employee in the employee text file but as soon as I include a second, the first is overwritten and the information for the second is what's created. This is some of the information I have so far:
class Staff:
staffData = {}
def __init__(self, EmpID, LName, FName, RHours, HRate, OTMult, TaxC, SBand, Date, HrsWorked):
self.EmpID = EmpID
self.LName = LName
self.FName = FName
self.RHours = RHours
self.HRate = HRate
self.OTMult = OTMult
self.TaxC = TaxC
self.SBand = SBand
self.Date = Date
self.HrsWorked = HrsWorked
Staff.staffData[EmpID] = self
def setup(staff ,hrs): # staff and hrs are text files
with open(staff) as empFile:
for line in empFile:
SID, LN, FN, RH, HR, OTM, TC, SB = line.split()
with open(hrs) as hFile:
for line in hFile:
SID, date, HrsWorked = line.split()
stf = Staff(SID, LN, FN, int(RH), int(HR), float(OTM), int(TC), int(SB),date, HrsWorked)
return stf
def paySlip(self,HrsWorked, date):
paymentDict = {}
#creates paySlip using information from instance and creates others by performing
calculations
return paymentDict
jg = Staff.setup('Staff.txt', 'Hrs.txt')
dic = jg.paySlip(42,'31/10/2021')
print(dic)
This is what I have so far, and it returns the correct output, but when I include a second line with a second staff member that is where I get the issues. Also, just a note, the parameters for paySlip have got to be date and hours worked from the hours text file and not just the text file. I'm quite new to programming so apologies if this is a basic fix, I've spent the last 3 days trying to figure it out by visiting different boards and websites. Any help would be greatly appreciated.
the line
return stf
return from the setup funktion after readin the first line no matter what. If you want to store all line in a list of Staff objects you need to build that list and then return it. something like that
with open(hrs) as hFile:
stf = []
for line in hFile:
SID, date, HrsWorked = line.split()
stf.append(Staff(SID, LN, FN, int(RH), int(HR), float(OTM), int(TC), int(SB),date, HrsWorked))
return stf
then you should get all lines in hFile. That said the way this is build is just stange. you build a staff object to breat a list of staff objects ?
you might wanna restructure your code into a Staff class and StaffMember class? this way you have a Staff object that has Members with hours. But i can be wrong here since i dont see the full picture
cheers
Markus
We have an application that is basically just a form submission for requesting a team drive to be created. It's hosted on Google App Engine.
This timeout error is coming from a single field in the form that simply does typeahead for an email address. All of the names on the domain are indexed in the datastore, about 300k entities - nothing is being pulled directly from the directory api. After 10 seconds of searching (via the Python Google Search API), it will time out. This is currently intermittent, but errors have been increasing in frequency.
Error: line 280, in get_result raise _ToSearchError(e) Timeout: Failed to complete request in 9975ms
Essentially, speeding up the searches will resolve. I looked at the code and I don't believe there is any room for improvement there. I am not sure if increasing the instance class will improve this, it is currently an F2. Or if perhaps there is another way to improve the index efficiency. I'm not entirely sure how one would do that however. Any thoughts would be appreciated.
Search Code:
class LookupUsersorGrpService(object):
'''
lookupUsersOrGrps accepts various params and performs search
'''
def lookupUsersOrGrps(self,params):
search_results_json = {}
search_results = []
directory_users_grps = GoogleDirectoryUsers()
error_msg = 'Technical error'
query = ''
try:
#Default few values if not present
if ('offset' not in params) or (params['offset'] is None):
params['offset'] = 0
else:
params['offset'] = int(params['offset'])
if ('limit' not in params) or (params['limit'] is None):
params['limit'] = 20
else:
params['limit'] = int(params['limit'])
#Search related to field name
query = self.appendQueryParam(q=query, p=params, qname='search_name', criteria=':', pname='query', isExactMatch=True,splitString=True)
#Search related to field email
query = self.appendQueryParam(q=query, p=params, qname='search_email', criteria=':', pname='query', isExactMatch=True, splitString=True)
#Perform search
log.info('Search initialized :\"{}\"'.format(query) )
# sort results by name ascending
expr_list = [search.SortExpression(expression='name', default_value='',direction=search.SortExpression.ASCENDING)]
# construct the sort options
sort_opts = search.SortOptions(expressions=expr_list)
#Prepare the search index
index = search.Index(name= "GoogleDirectoryUsers",namespace="1")
search_query = search.Query(
query_string=query.strip(),
options=search.QueryOptions(
limit=params['limit'],
offset=params['offset'],
sort_options=sort_opts,
returned_fields = directory_users_grps.get_search_doc_return_fields()
))
#Execute the search query
search_result = index.search(search_query)
#Start collecting the values
total_cnt = search_result.number_found
params['limit'] = len(search_result.results)
#Prepare the response object
for teamdriveDoc in search_result.results:
teamdriveRecord = GoogleDirectoryUsers.query(GoogleDirectoryUsers.email==teamdriveDoc.doc_id).get()
if teamdriveRecord:
if teamdriveRecord.suspended == False:
search_results.append(teamdriveRecord.to_dict())
search_results_json.update({"users" : search_results})
search_results_json.update({"limit" : params['limit'] if len(search_results)>0 else '0'})
search_results_json.update({"total_count" : total_cnt if len(search_results)>0 else '0'})
search_results_json.update({"status" : "success"})
except Exception as e:
log.exception("Error in performing search")
search_results_json.update({"status":"failed"})
search_results_json.update({"description":error_msg})
return search_results_json
''' Retrieves the given param from dict and adds to query if exists
'''
def appendQueryParam(self, q='', p=[], qname=None, criteria='=', pname=None,
isExactMatch = False, splitString = False, defaultValue=None):
if (pname in p) or (defaultValue is not None):
if len(q) > 0:
q += ' OR '
q += qname
if criteria:
q += criteria
if defaultValue is None:
val = p[pname]
else:
val = defaultValue
if splitString:
val = val.replace("", "~")[1: -1]
#Helps to retain passed argument as it is, example email
if isExactMatch:
q += "\"" +val + "\""
else:
q += val
return q
An Index instance's search method accepts a deadline parameter, so you could use that to increase the time that you are willing to wait for the search to respond:
search_result = index.search(search_query, deadline=30)
The documentation doesn't specify acceptable value for deadline, but other App Engine services tend to accept values up to 60 seconds.
I have a file of constant variables that I need to query and I am not sure how to go about it.
I have a database query which is returning user names and I need to find the matching user name in the file of constant variables.
The file looks like this:
SALES_MANAGER_01 = {"user_name": "BO01", "password": "password", "attend_password": "BO001",
"csm_password": "SM001", "employee_num": "BOSM001"}
There is just a bunch of users just like the one above.
My function looks like this:
#attr("user_test")
def test_get_user_for_login(self):
application_code = 'BO'
user_from_view = self.select_user_for_login(application_code=application_code)
users = [d['USER'] for d in user_from_view]
user_with_ent = choice(users)
user_wo_ent = user_with_ent[-4:]
password = ""
global_users = dir(gum)
for item in global_users:
if user_wo_ent not in item.__getattr__("user_name"):
user_with_ent = choice(users)
user_wo_ent = user_with_ent[-4:]
else:
password = item.__getattr__("password")
print(user_wo_ent, password)
global_users = dir(gum) is my file of constants. So I know I am doing something wrong since I am getting an attribute error AttributeError: 'str' object has no attribute '__getattr__', I am just not sure how to go about resolving it.
You should reverse your looping as you want to compare each item to your match condition. Also, you have a dictionary, so use it to do some heavy lifting.
You need to add some imports
import re
from ast import literal_eval
I've changed the dir(gum) bit to be this function.
def get_global_users(filename):
gusers = {} # create a global users dict
p_key = re.compile(ur'\b\w*\b') # regex to get first part, e.g.. SALES_MANAGER_01
p_value = re.compile(ur'\{.*\}') # regex to grab everything in {}
with (open(filename)) as f: # open the file and work through it
for line in f: # for each line
gum_key = p_key.match(line) # pull out the key
gum_value = p_value.search(line) # pull out the value
''' Here is the real action. update a dictionary
with the match of gum_key and with match of gum_value'''
gusers[gum_key.group()] = literal_eval(gum_value.group())
return(gusers) # return the dictionary
The bottom of your existing code is replaced with this.
global_users = get_global_users(gum) # assign return to global_users
for key, value in global_users.iteritems(): # walk through all key, value pairs
if value['user_name'] != user_wo_ent:
user_with_ent = choice(users)
user_wo_ent = user_with_ent[-4:]
else:
password = value['password']
So a very simple answer was get the dir of the constants file then parsing over it like so:
global_users = dir(gum)
for item in global_users:
o = gum.__dict__[item]
if type(o) is not dict:
continue
if gum.__dict__[item].get("user_name") == user_wo_ent:
print(user_wo_ent, o.get("password"))
else:
print("User was not in global_user_mappings")
I was able to find the answer by doing the following:
def get_user_for_login(application_code='BO'):
user_from_view = BaseServiceTest().select_user_for_login(application_code=application_code)
users = [d['USER'] for d in user_from_view]
user_with_ent = choice(users)
user_wo_ent = user_with_ent[4:]
global_users = dir(gum)
user_dict = {'user_name': '', 'password': ''}
for item in global_users:
o = gum.__dict__[item]
if type(o) is not dict:
continue
if user_wo_ent == o.get("user_name"):
user_dict['user_name'] = user_wo_ent
user_dict['password'] = o.get("password")
return user_dict
Given PostgreSQL 9.2.10, Django 1.8, python 2.7.5, the following model:
class soapProdAPI(models.Model):
soap_id = models.PositiveIntegerField(primary_key=True)
soap_host = models.CharField(max_length=20)
soap_ip = models.GenericIPAddressField(default='0.0.0.0')
soap_asset = models.CharField(max_length=20)
soap_state = models.CharField(max_length=20)
And the following code:
tableProdSoap = soapProdQuery()
#periodic_task(run_every=timedelta(minutes=2))
def saveSoapProd():
tableProdSoap = soapProdQuery()
if tableProdSoap != None:
for item in tableProdSoap:
commit = soapProdAPI(soap_id=item[0], soap_host=item[1], soap_asset=item[2], soap_state=item[3])
commit.save()
saveSoapNullIP()
To answer Josué Padilla's question:
#task
def saveSoapNullIP():
missingIP = soapProdAPI.objects.filter(soap_ip='0.0.0.0')
if missingIP:
for record in missingIP:
if str(record.soap_host).lower().startswith('1a'):
fqdn = str(record.soap_host) + 'stringvaluehere'
elif str(record.soap_host).lower().startswith('1b'):
fqdn = str(record.soap_host) + 'stringvaluehere'
elif str(record.soap_host).lower().startswith('1c'):
fqdn = str(record.soap_host) + 'stringvaluehere'
else:
fqdn = str(record.soap_host) + 'stringvaluehere'
try:
hostIp = check_output('host %s' % fqdn, shell=True)
hostIp = hostIp.split()[-1]
except:
hostIp = '0.0.0.0'
record.soap_ip = hostIp
record.save(update_fields=['soap_ip'])
My soapProdQuery only returns these 4 fields where there is a 5th field in the model (soap_ip). I know it is probably not the best way to do it but I have a separate block of code that queries the db for None values in soap_ip runs a subprocess host on them and saves it back with the ip address (The number of rows returned/updated should get smaller each pass through, as opposed to putting the logic for doing a host lookup into the request/this celery task itself which would run every API request. I have tried this already, it takes FOREVER to return the completed data.). The soap API I query does not provide the IP or I would grab it that way obviously. This all runs as background tasks using celery to make it invisible/seamless to the web user.
The issue I run into is that every time the saveSoapProd() runs it overwrites the previous soap_ip field with '0.0.0.0' thus negating the work of my other function. The other issue is that I cannot force_insert or force_update as I need both functionalities with this. My question is this: is there a way to selectively update/insert at the same time and completely exclude doing anything to the soap_ip each time saveSoapProd() runs? Any and all help is greatly appreciated. Thank you in advance.
** EDIT 1 **
I may or may not have found a solution in update_or_create or get_or_create, however I am unsure on the exact usage. The docs have me slightly confused.
** EDIT 2 **
I guess get_or_create is a bust. Works first pass through but every save after that fails with this:
Traceback (most recent call last):
File "<console>", line 1, in <module>
File "<console>", line 8, in saveSoapProd
File "/lib/python2.7/site-packages/django/db/models/base.py", line 690, in save
% ', '.join(non_model_fields))
ValueError: The following fields do not exist in this model or are m2m fields: soap_id
Here is the code:
#periodic_task(run_every=timedelta(minutes=2))
def saveSoapProd():
tableProdSoap = soapProdQuery()
if tableProdSoap != None:
for item in tableProdSoap:
obj, created = soapProdAPI.objects.get_or_create(soap_id=item[0], defaults={'soap_host': item[1], 'soap_asset': item[2], 'soap_state': item[3]})
if created == False:
commit = soapProdAPI(soap_id=item[0], soap_host=item[1], soap_asset=item[2], soap_state=item[3])
commit.save(update_fields=['soap_id', 'soap_host', 'soap_asset', 'soap_state'])
I will be honest, I am not entirely sure what is causing this error.
** EDIT 3/CURRENT SOLUTION **
I was able to resolve my own issue by modifying my model and my task function. The solution uses get_or_create, but you could easily extrapolate how to use update_or_create from the solution provided. See the selected answer below for a coded example.
** TLDR **
I want to do a .save() where it may need to do a insert for new records or update for changed records WITHOUT touching the soap_ip field (no insert_only or update_only).
I don't know if you already knew this, but you can override the save() function of your model.
class soapProdAPI(models.Model):
soap_id = models.PositiveIntegerField(primary_key=True)
soap_host = models.CharField(max_length=20)
soap_ip = models.GenericIPAddressField(default='0.0.0.0')
soap_asset = models.CharField(max_length=20)
soap_state = models.CharField(max_length=20)
# Override save
def save(self, *args, **kwargs):
if self.soap_ip != '0.0.0.0':
self.soap_ip = your_ip # Here you can get your old IP an save that instead of 0.0.0.0
EDIT
You are getting
ValueError: The following fields do not exist in this model or are m2m fields: soap_id
Because you are trying to update soap_id, that field is defined as your model's primary key, so it is immutable when updating. That's why it crashes when you do:
commit.save(update_fields=['soap_id', 'soap_host', 'soap_asset', 'soap_state'])
Try removing soap_id from update_fields.
Solved my own issue without modifying the save method by making the following changes to my model:
class soapProdAPI(models.Model):
soap_id = models.PositiveIntegerField(unique=True, null=False)
soap_host = models.CharField(max_length=20)
soap_ip = models.GenericIPAddressField(default='0.0.0.0')
soap_asset = models.CharField(max_length=20)
soap_state = models.CharField(max_length=20)
and my task:
def saveSoapProd():
tableProdSoap = soapProdQuery()
if tableProdSoap != None:
for item in tableProdSoap:
try:
obj, created = soapProdAPI.objects.get_or_create(soap_id=item[0], defaults={'soap_host': item[1], 'soap_asset': item[2], 'soap_state': item[3]})
if created == False:
obj.soap_host = item[1]
obj.soap_asset = item[2]
obj.soap_state = item[3]
obj.save(update_fields=['soap_host', 'soap_asset', 'soap_state'])
except:
continue
saveSoapMissingIP()
EDIT
Just noticed Josué Padilla's response, which was in fact part of my problem that I solved with this answer. Thank you to Josué for all of your help.
I have a hundreds of pages of "quiz" questions, multiple-choice options and associated answer keys and explanations. I'm trying to create a simple Django app to administer these questions. I have created a simple but effective Python parser to parse the scanned, OCR'd pages into the proper objects.
I want to have a "utility" to enable the administrator of this Django app to import quiz content from OCR'd paper into the relevant Django DB tables. This will be a rare task, and something that is not necessarily appropriate for inclusion in a web UI.
I've asked about using intermediate JSON/YAML fixtures and was told that a more appropriate approach would be to directly create and save instances of my models [1]. I then tried to create a standalone script along the lines suggested by [2] and [3] but was unable to overcome the
kwargs = {"app_label": model_module.__name__.split('.')[-2]}
IndexError: list index out of range
error.
I also came across [4] about creating custom django-admin.py/manage.py commands. This seems like a logically appropriate way of dealing with the task; but, I'd love to hear from those with more experience and brains (I've eaten all mine :).
References:
Importing data from scanned text into Django as YAML fixture or SQL
what is the simplest way to create a table use django db api ,and base on 'Standalone Django scripts'
Standalone Scripts
Writing custom django-admin commands
Examples:
OCR'd Text
Page 12
34. Hiedegger is a _____ . (a) philosopher (b) boozy beggar (c) both a and b (d) none of these 35. ...
Django Models
class Question(models.Model):
text = models.TextField()
class Choice(models.Model):
question = models.ForeignKey(Question)
order = models.IntegerField(default=1)
text = models.TextField()
Goal, something like this...
q = Question.objects.create(text="Hiedegger is a _____ .")
q.save()
c = Choice(text="philosopher", order=1, question=q.pk)
c.save()
This is the working version I came up with. Dirty, but effective. Both #akonsu and #Ivan Kharlamov were helpful. Thanks...
import os, re, Levenshtein as lev, codecs
from SimpleQuiz.quiz.models import Choice, Question
from django.core.management.base import BaseCommand, CommandError
import optparse
class Command(BaseCommand):
args = '--datapath=/path/to/text/data/'
can_import_settings = True
help = 'Imports scanned text into Questions and Choices'
option_list = BaseCommand.option_list + (
optparse.make_option('--datapath', action='store', type='string',
dest='datapath',
help='Path to OCRd text files to be parsed.'),
)
requires_model_validation = True
# Parser REs
BACKUP_RE = re.compile(r'\~$|bak$|back$|backup$')
QUEST_RE = re.compile(r'^[0-9]{1,3}[.][ ]')
CHOICE_RE = re.compile(r'^[a-e][.][ ]')
def handle(self, *args, **options):
# get the data path
try:
os.path.exists(options['datapath'])
except Exception as e:
raise CommandError("None or invalid path provided: %s" % e.message)
self.datapath = os.path.expanduser(options['datapath'])
# generate list of text strings from lines in target files
self.data_lines = []
for fn in os.listdir(os.path.join(self.datapath, 'questions/')):
if self.BACKUP_RE.search(fn):
self.stderr.write("Skipping backup: %s\n" % (fn))
else:
for line in codecs.open(os.path.join(self.datapath, 'questions/', fn), 'r', encoding='latin-1'):
if not self.is_boilerplate(line):
if not line.strip() == '':
self.data_lines.append(line)
#-----------------------------------------------------------------------
#--------------------- Parse the text lines and create Questions/Choices
#-----------------------------------------------------------------------
cur_quest = None
cur_choice = None
cur_is_quest = False
questions = {}
choices = {}
for line in self.data_lines:
if self.is_question(line):
[n, txt] = line.split('.', 1)
qtext = txt.rstrip() + " "
q = Question.objects.create(text=qtext)
q.save()
cur_quest = q.pk
questions[cur_quest] = q
cur_is_quest = True
elif self.is_choice(line):
[n, txt] = line.split('.', 1)
num = self.char2dig(n)
ctext = txt.rstrip() + " "
c = Choice.objects.create(text=ctext, order=num, question=questions[cur_quest])
c.save()
cur_choice = c.pk
choices[cur_choice] = c
cur_is_quest = False
else:
if cur_is_quest:
questions[cur_quest].text += line.rstrip() + " "
questions[cur_quest].save()
else:
choices[cur_choice].text += line.rstrip() + " "
choices[cur_choice].save()
self.stdout.write("----- FINISHED -----\n")
return None
def is_question(self, arg_str):
if self.QUEST_RE.search(arg_str):
return True
else:
return False
def is_choice(self, arg_str):
if self.CHOICE_RE.search(arg_str):
return True
else:
return False
def char2dig(self, x):
if x == 'a':
return 1
if x == 'b':
return 2
if x == 'c':
return 3
if x == 'd':
return 4
if x == 'e':
return 5
def is_boilerplate(self, arg_str):
boilerplate = [u'MFT PRACTICE EXAMINATIONS',
u'BERKELEY TRAINING ASSOCIATES ' + u'\u00A9' + u' 2009',
u'BERKELEY TRAINING ASSOCIATES',
u'MARRIAGE AND FAMILY THERAPY',
u'PRACTICE EXAMINATION 41',
u'Page 0', u'Page 1', u'Page 2', u'Page 3', u'Page 4',
u'Page 5', u'Page 6', u'Page 7', u'Page 8', u'Page 9',
]
for bp in boilerplate:
if lev.distance(bp.encode('utf-8'), arg_str.encode('utf-8')) < 4:
return True
return False
I then tried to create a standalone script along the lines suggested
by [2] and [3] but was unable to overcome the kwargs = {"app_label":
model_module.name.split('.')[-2]} IndexError: list index out of
range error.
I had the same list index error. It was caused by the way I imported the models in my script. I used to do it like this:
from models import Table1, Table2
Then I realized the Python script is not part of the application, so I changed the import to:
from myapp.models import Table1, Table2
My Python script is started with the following shell script:
export DJANGO_SETTINGS_MODULE=settings
export PYTHONPATH=/path/to/my/site
python myscript.py "$#"