Develop Reference

Unable to import Excel to database

I have a system whereby it allows to import the student attendance file into the system. However, it did not import it successfully. I tried to debug and i found out that when it goes to the importer.py function, it does not go to the "def _do_save(self, row)", hence that is the problem why it was not saved into database.
importer.py
class AttendanceImporter(CsvImporter):
field_names=["username", "mark"]
#it does not go to the method.
def _handle_row(self,row):
print("this")
if (not self._is_row_valid(row)):
return self._FAILED
username=row["username"]
if (self._is_username_exist(username)):
if (self._is_mark_exist(username)):
if (self._do_update(row)):
return self._UPDATED
else:
return self._FAILED
else:
if (self._do_save(row)):
return self._CREATED
else:
return self._FAILED
else:
return self._FAILED
def _is_row_valid(self, row):
for item in self.field_names:
if (len(row[item])==0):
return False
return True
def _is_username_exist(self, username):
return len(User.objects.filter(username=username))>0
print(username)
def _is_mark_exist(self, username):
user = User.objects.get(username=username)
return len(Attendance.objects.filter(user=user))>0
print(username)
def _do_save(self, row):
# create attendace mark
try:
attendance=Attendance()
user=User.objects.get(username=row["username"])
attendance=Attendance.objects.create(user=user, mark=row["mark"])
print("save?")
attendance.save()
except:
return False
return True
def _do_update(self, row):
# update attendance mark
try:
user=User.objects.get(username=row["username"])
attendance=Attendance.objects.get(user=user)
attendance.mark = row["mark"]
attendance.save()
except Exception as e:
print(e)
return False
return True
Views.py
#transaction.atomic
#csrf_exempt
def data_import(request, file_type):
# TODO: currently only support importing users, later can support importing groups
fields_required = None
if(file_type == "user"):
fields_required = "username, password, email, matric_number, fullname, groups"
elif (file_type == "attendance"):
fields_required = "username, mark"
if request.FILES:
successful = False
try:
im = None
if(file_type == "user"):
upload_file = request.FILES['user_file']
file_path = save_uploaded_file(request.FILES['file'], filename=generate_unique_file_name(extension="csv"),
filedir=USER_DATA_UPLOAD_PATH)
im = StudentImporter(source=open(file_path))
elif (file_type == "attendance"):
upload_file = request.FILES['attendance_file']
file_path = save_uploaded_file(upload_file, filename=generate_unique_file_name(extension="csv"),
filedir=USER_DATA_UPLOAD_PATH)
im = AttendanceImporter(source=open(file_path))
successful, result = im.import_to_database()
except Exception:
pass
if successful:
messages.info(request, "The import is successful!\n" + result)
else:
messages.warning(request, "The import is NOT successful, no data is imported!")
return HttpResponseRedirect(reverse("student_user_profile_list"))
return render(request,
"app-appglobal/import-data.html",
{'type': file_type, 'fields_required': fields_required})
Model.py:
class Attendance(models.Model):
user=models.OneToOneField(User, on_delete=models.CASCADE)
mark=models.IntegerField(default=0)
class Meta:
ordering=['user']
def get_mark(self):
return self.mark
Urls
url(r'^student/attendance/$' ,views.data_import,{'file_type':'attendance'},name='attendance_import'),
CSVIMPORTER
class CsvImporter(object, metaclass=ABCMeta):
def __init__(self, source):
"""
:param source: a file object
:return:
"""
self.source=source
def import_to_database(self):
"""
:return: (successful:boolean,result:string)
"""
pass
example:

Good way to writing unit test in python

I have simply 'to do' app in python 2.7 and I wrote for this some unit test. This is my first time with python unit test and I just wanna to know idea of unit test.
Someone can tell me whether I'm going in the right direction?
How to improve this tests?
How to check message in IndexError is correct? for this ("IndexError('Note doesn\'t exist')" or IndexError('Returned more then one entry') )
App:
# coding: utf-8
from __future__ import unicode_literals
from shutil import copyfile
import json
import os
DATABASE = 'notes_data/notes.json'
BOARDS = ['to do', 'in progress', 'done']
class NotesManagerMixin(object):
def count(self):
return len(self.notes)
def filter(self, *args, **kwargs):
result = self.notes
for key, value in kwargs.iteritems():
result = [
note for note in result
if getattr(note, key, None) == value or
note.message.startswith(str(value)) or
note.message.endswith(str(value))
]
return NotesQueryset(result)
def get(self, *args, **kwargs):
notes = self.filter(*args,**kwargs)
if notes.count() == 0:
raise IndexError('Note doesn\'t exist')
elif notes.count() == 1:
return notes[0]
else:
raise IndexError('Returned more then one entry')
def first(self):
return self.notes[0]
def last(self):
return self.notes[-1]
class NotesQueryset(NotesManagerMixin):
def __init__(self, notes):
self.notes = [note for note in notes]
def update(self, *args, **kwargs):
for note in self.notes:
for key, value in kwargs.items():
setattr(note, key, value)
note.save()
return self
def delete(self):
for note in self.notes:
note.delete()
return self
def __getitem__(self, idx):
return self.notes[idx]
def __str__(self):
return str(self.notes)
def __repr__(self):
return self.__str__()
class NotesManager(NotesManagerMixin):
def __init__(self):
self.notes = []
def __iter__(self):
return self.next()
def __generate_id(self):
"""
Funkcja pomocnicza do pobrania pewnej wolnej wartości indexu.
"""
try:
return max(note.id for note in self.notes) + 1
except ValueError:
return 1
def all(self):
return NotesQueryset(self.notes)
def add(self, idx, board, message):
self.notes.append(Note(idx=idx, board=board, message=message))
def create(self, board, message):
note = Note(
idx=self.__generate_id(),
board=board,
message=message
)
note.clean()
self.notes.append(note)
note.save()
return note
def next(self):
for note in self.notes:
yield note
def to_dict(self):
return [note.to_dict() for note in self.notes]
class Note(object):
objects = NotesManager()
def __init__(self, idx, board, message):
self.id = idx
self.board = board
self.message = message
def __str__(self):
return 'ID: {}, Board: {}, Message: {}'.format(
self.id,
self.board,
self.message
)
def __repr__(self):
return self.__str__()
def clean(self):
if not self.message:
raise ValueError('Message is required')
if self.board not in BOARDS:
raise ValueError('Board "{}" doesn\'t exists'.format(self.board))
if type(self.id) != int:
raise ValueError('Note id "{}" is invalid'.format(self.id))
def save(self):
for key, note in enumerate(self.objects):
if note.id == self.id:
self.objects.notes[key] = self
break
with open(DATABASE, 'w') as database_file:
json.dump(self.objects.to_dict(), database_file, indent=4)
return True
def delete(self):
for key, note in enumerate(self.objects.notes):
if note.id == self.id:
self.objects.notes.pop(key)
with open(DATABASE, 'w') as database_file:
json.dump(self.objects.to_dict(), database_file, indent=4)
def to_dict(self):
return {
'id': self.id,
'message': self.message,
'board': self.board
}
def load_initial_data():
with open(DATABASE, 'r') as database_file:
json_data = json.load(database_file, encoding='utf-8')
for item in json_data:
Note.objects.add(
idx=item['id'],
board=item['board'],
message=item['message'],
)
load_initial_data()
unit tests:
import unittest
from notes_manager_v2 import NotesQueryset, Note, load_initial_data, NotesManagerMixin
class TestNotesQueryset(unittest.TestCase):
def test_filter(self):
actual = Note.objects.all().filter(board='in progress') # get all notes with board 'in progress'
expected = []
for note in Note.objects.all():
if note.board == 'in progress':
expected.append(note)
self.assertItemsEqual(actual, expected)
def test_get(self):
actual = [Note.objects.all().get(id=1)] # get note with method get
expected = []
for note in Note.objects.all(): # search note with index 1
if note.id == 1:
expected.append(note)
self.assertEqual(actual, expected)
def test_get_fail_1(self):
self.assertRaises(IndexError, lambda:Note.objects.all().get(id=9868976)) # thos note dont exist should raise IndexError
def test_update(self):
from_board = 'to do'
to_board = 'done'
before_change = Note.objects.filter(board=from_board) # use filter method to get all notes with board 'to do'
actual = Note.objects.filter(board=from_board).update(board=to_board) # update all board
self.assertNotEqual(before_change, actual) # check for difference
notes = Note.objects.all()
for note in actual:
self.assertIn(note, notes) # check notes are updated
def test_delete(self):
to_delete = Note.objects.filter(id=2).delete() # find note with filter method and delete it
notes = Note.objects.all()
self.assertNotIn(to_delete, notes)
def test_create(self):
new_note = Note.objects.create(message='lorem ipsum', board='in progress') # create new note
notes = Note.objects.all()
self.assertIn(new_note, notes) #
if __name__ == '__main__':
unittest.main()

Have you looked at the documentation? See doctest. They are an easy way to integrate unit tests into python code. Another option is the unittest framework.

codernitydb string key whats wrong with this code?

from CodernityDB.database import Database
from CodernityDB.hash_index import HashIndex
class WithXIndex(HashIndex):
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '16s'
super(WithXIndex, self).__init__(*args, **kwargs)
def make_key_value(self, data):
username = data['username']
# if not isinstance(login, basestring):
# login = str(login)
return md5(username).digest(), None
def make_key(self, key):
return md5(key).digest()
def main():
db = Database('l1.db')
if db.exists():
db.open()
else:
db.create()
x_ind = WithXIndex(db.path, 'username')
db.add_index(x_ind)
db.insert(dict( username='lamar', age='33', frm='new jersey'))
for dt in db.all('username',with_doc=True):
print dt
print db.get('username', 'lamar') # throws an exception

Cherrypy with Mysql error: "tuple indices must be integers, not str" when trying to add users

guys, I'm having this random error on my Project when trying to register users.
This is the information that is returned from the browser.
{Gender: "M", ReEmail: "heyguys#whatup.com", Birthday: "1/19/1980", Role: "Athlete",…}
Birthday: "1/19/1980"
Country: "United States"
Email: "heyguys#whatup.com"
Gender: "M"
Password:"doesntwork123"
ReEmail:"heyguys#whatup.com"
RePassword:"doesntwork123"
Role:"Athlete"
kwargs:{}
method:"POST"
results:{error: "tuple indices must be integers, not str"}
error: "tuple indices must be integers, not str"
By using PyCharm debugger, I have centralized the problem to this specific part
def addUser(self, session, *args, **kwargs):
try:
#existingPerson = session.query(person_models.Player).filter(person_models.Player.Email==args['Email']).first()
print person_models.Player
print person_models.Player.Email
existingPerson = session.query(person_models.Player).filter(person_models.Player.Email == args['Email']).first()
print existingPerson
if existingPerson is not None:
return {'error':"User already exists"}
person_model = getattr(person_models)
person = person_model(*args)
session.add(person)
session.commit()
return {"response": "User added"}
except Exception as e:
return {'error':str(e)}
Once it gets to existing person, it sends me directly to the exception. This is when it gives me "tuple indices must be integers, not str"
Here is some additional code.
Here is some code:
webapi.py
import cherrypy
from api.restBase import jsonDbRest
from person.controllers import PersonController
class PersonAPI:
exposed = True
#jsonDbRest
def GET(self, email=False, **kwargs):
session = cherrypy.request.db
pc = PersonController()
if email in kwargs:
profile = pc.get(session, **kwargs)
return {"results":[profile], "count":'1'}
profile = pc.search(session, **kwargs)
return {"results":profile.to_dict(), "count":'1'}
#jsonDbRest
def POST(self, *args, **kwargs):
session = cherrypy.request.db
pc = PersonController()
response = pc.addUser(session, *args, **kwargs)
return {"results": response}
It's complaining about line 15.
Here is another line that is using the code:
restbase.py
import cherrypy
import json
from engine import create_session
def jsonDbRest(func):
def wrapper(self, *args, **kwargs):
cherrypy.request.db = create_session()
if 'Content-Length' in cherrypy.request.headers:
length = cherrypy.request.headers['Content-Length']
bodytext = cherrypy.request.body.fp.read(int(length))
else:
bodytext = ""
if bodytext != "":
jsonData = json.loads(bodytext)
else:
jsonData = None
cherrypy.request.json = jsonData
func_data = func(self, *args, **kwargs)
data = {"method":func.__name__, "input":jsonData,"kwargs":kwargs}
if func_data is not None:
data.update(func_data)
cherrypy.response.headers['Content-Type'] = 'application/json'
if "error" in data:
cherrypy.response.status = "400"
else:
cherrypy.response.status = "200"
cherrypy.request.db.close()
return json.dumps(data, indent=4)
return wrapper
Another code
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.declarative import AbstractConcreteBase
import datetime
import decimal
import netaddr
import sqlalchemy
from time import strftime
class Base(object):
def to_dict(self):
dict = {}
dict = self.__dict__
remove = '_sa_instance_state'
dict = {key: value for key, value in dict.items() if key is not remove}
dict['DateOfBirth'] = dict['DateOfBirth'].strftime('%m/%d/%Yi')
return dict
def from_dict(self, dict):
for col in dict:
setattr(self, col, dict[col])
def to_array(self, columns=[]):
if len(columns) < 1:
columns = self.__mapper__.columns.keys()
data = []
for col in columns:
data.append(getattr(self, col))
return data
def get_columns(self):
return self.__mapper__.columns.keys()
JsonBase = declarative_base(cls=Base)

I think that your problem is on the first sample:
existingPerson = session.query(person_models.Player).filter(person_models.Player.Email == args['Email']).first()
Try with kwargs instead of args:
existingPerson = session.query(person_models.Player).filter(person_models.Player.Email == kwargs['Email']).first()

combine class instance into another instance in python?

I am in designing one "jigsaw puzzle" like tool to manage different water pipe like parts combination for fun.
I have different single parts type with different purpose(cooler, heater, purifier...)
Those parts with different interfaces size can be connected with each other (1/4 inch. 1/6 inch ....)
I want those parts can be stored in database and can be combined into a total new parts combination(randomly or purposely), but still can be considering as a function-able part.
Here is the initial thinking
class MetaInfo():
def __init__(self, name, intype,outtype,shape,serialno):
this.name = name
this.intype = intype
this.outtype = outtype
this.shape = shape
this.sn = serialno
def parts():
def __init__(self, meta):
this.meta = meta
def linkwith(self, part):
if part.meta.shape == this.meta.shape:
# make it simple, logical here is just same shape can be connected each other
return ??? # a new parts combination
else:
raise Error
m1 = MetaInfo("cooler", "hotwater", "coldwater", "1/4 inch round", "SN:11111" )
m2 = MetaInfo("heater", "coldwater", "hotwater", "1/4 inch round", "SN:22222" )
m3 = MetaInfo("purifier", "coldwater", "hotwater", "1/6 inch round", "SN:33333" )
a = parts(m1)
b = parts(m2)
c = parts(m3)
Here is what I need your help:
how to save m1, m2, m3 as a list which can persistent in a human readable database, next time only change that database itself I can add meta?
how to chain different parts as a new combination? Such as
e = a.linkwith(b)
d = c.linkwith(a)
and store it in that database as well?
can I make a long chain, make a new parts instance , such as
f = c.linkwith(a,b,d,e)
and findout easily which part is incapable to link in this chain, here part c with different size?
Many thanks.

I got bored. It's very rough but it works. If you take this far enough, you will want to use a database; but I understand wanting to use a human readable format.
from copy import copy
import csv
class Part_Base(object):
pass
class MultiPart_Base(list):
pass
class part_meta(type):
part_names = {}
parts = []
def __init__(cls, cls_name, cls_bases, cls_dict):
super(part_meta, cls).__init__(cls_name, cls_bases, cls_dict)
if(not Part_Base in cls_bases):
part_meta.part_names[cls_name] = cls
def __call__(self, *args, **kwargs):
name = kwargs.get("name", "")
if(part_meta.part_names.has_key(name) and not (self is part_meta.part_names[name])):
obj = part_meta.part_names[name].__call__(*args, **kwargs)
else:
obj = None
if(not part_meta.part_names.has_key(self.__name__)):
new_class = part_meta(name, (Generic_Part,), {})
globals()[name] = new_class
obj = new_class(*args, **kwargs)
else:
obj = super(part_meta, self).__call__(*args, **kwargs)
if not obj in part_meta.parts:
part_meta.parts.append(obj)
return obj
#classmethod
def save(cls):
all_fields = list(reduce(lambda x, y: x | set(y.fields), cls.parts, set([])))
with open("parts.csv", "w") as file_h:
writer = csv.DictWriter\
(
file_h,
all_fields,
restval = "",
extrasaction = "ignore",
dialect = "excel",
lineterminator = "\n",
)
writer.writeheader()
for part in cls.parts:
writer.writerow({field : getattr(part, field) for field in part.fields})
#classmethod
def load(cls):
with open("parts.csv", "r") as file_h:
reader = csv.DictReader(file_h)
for row in reader:
Part(**row)
class Part(Part_Base):
__metaclass__ = part_meta
fields = []
def __init__(self, **kwargs):
for name, value in kwargs.items():
setattr(self, name, value)
self.fields += kwargs.keys()
def __repr__(self):
return "<%s>" % self.description
#property
def description(self):
return "%s: %s %s %s %s" % (self.name, self.intype, self.outtype, self.shape, self.serialno)
def linkwith(self, *parts):
return Generic_MultiPart(self, *parts)
class Generic_Part(Part):
def __init__(self, **kwargs):
kwargs["name"] = self.__class__.__name__
super(Generic_Part, self).__init__(**kwargs)
class Generic_MultiPart(MultiPart_Base):
def __init__(self, *parts):
super(Generic_MultiPart, self).__init__()
if len(parts) >= 2:
self.shape = parts[0].shape
self.linkwith(*parts)
else:
raise ValueError("Not enough parts")
def __repr__(self):
return "<MultiPart: %s>" % super(Generic_MultiPart, self).__repr__()
def linkwith(self, *parts):
for part in parts:
if part.shape == self.shape:
if isinstance(part, Part):
self.append(part)
elif isinstance(part, MultiPart_Base):
self.extend(part)
else:
raise ValueError("Incompatible parts")
return self
class cooler(Generic_Part):
intype = "hotwater"
outtype = "coldwater"
fields = ["intype", "outtype"]
class heater(Generic_Part):
intype = "coldwater"
outtype = "hotwater"
fields = ["intype", "outtype"]
def make_some_parts():
some_parts = \
[
# This is actually a cooler object
# The metaclass uses the cooler class from above
# to create the object
Part
(
name = "cooler",
shape = "1/4 inch round",
serialno = "SN:11111"
),
# Using the heater class directly
heater
(
shape = "1/4 inch round",
serialno = "SN:22222"
),
Part
(
name = "purifier",
intype = "coldwater",
outtype = "hotwater",
shape = "1/6 inch round",
serialno = "SN:33333"
),
Part
(
name = "carbon_filter",
intype = "coldwater",
outtype = "coldwater",
shape = "1/4 inch round",
serialno = "SN:33333"
)
]
useless_part = some_parts[0].linkwith(some_parts[1])
print useless_part
filter_part = copy(useless_part).linkwith(some_parts[3])
print filter_part
part_meta.save()
def load_some_parts():
part_meta.load()
print part_meta.parts
You can manually edit parts.csv (in Excel or other) and it will make the parts described.
The save/restore functionality hasn't been extended to MultiParts; you can do that.