Main idea of these work is to parse each page from pagination(it has 152 pages), and put these data to db. while running error occurred, column 'execution_period' cannot be null".
in my DataFrame I did this:
df['Срок исполнения'] = df['Срок исполнения'].fillna(0)
df['Дата принятия решения/дела'] = df['Дата принятия решения/дела'].fillna(0)
don't know how to fix it.
df = pd.DataFrame()
df_list = pd.read_html('sanction.xls')
df = df_list[0]
database = MySQLdb.connect (host="host", user = "root", passwd = "password", db = "db", charset='utf8')
mycursor = database.cursor()
cursor = database.cursor()
query = """INSERT INTO sanction (
id,
organization_type, organization, date,
decision_number, penalty_type, penalty_way,
penalty, violation, execution_period,
article, note, type_npa,
department, uploaded_date)
VALUES (
null,
%s, %s, STR_TO_DATE(%s,"%%d.%%m.%%Y"),
%s, %s, %s,
%s, %s, STR_TO_DATE(%s,"%%d.%%m.%%Y"),
%s, %s, %s,
%s, %s)"""
for r in range(1, sheet.nrows):
organization_type = sheet.cell(r,1).value
organization = sheet.cell(r,2).value
date = (sheet.cell(r,3).value)
decision_number = sheet.cell(r,4).value
penalty_type = sheet.cell(r,5).value
penalty_way = sheet.cell(r,6).value
penalty = sheet.cell(r,7).value
violation = sheet.cell(r,8).value
execution_period = sheet.cell(r,9).value
article =sheet.cell(r,10).value
note =sheet.cell(r,11).value
type_npa =sheet.cell(r,12).value
department =sheet.cell(r,13).value
uploaded_date =datetime.now().strftime("%Y-%m-%d %H:%M")
values = (organization_type, organization, date, decision_number, penalty_type,
penalty_way,penalty, violation, execution_period,article, note, type_npa, department,uploaded_date)
mycursor.execute(query, values)
Related
A new member has been registered in the database, but when entering the profile modification page and adding other information, it is registered in a new row and not in the same information as the registered member.
register new member :
def add_users(self):
add_fullname = self.line_fullname.text()
add_username = self.line_username.text()
add_Email = self.line_email.text()
add_password = self.line_password.text()
add_phone = self.line_telephon.text()
add_birthday = self.dateofbirth.text()
add_profession = self.line_profession.text()
add_Country = self.line_country.text()
add_keyone = self.keyone.text()
add_keytwo = self.keytwo.text()
add_newpassword = self.new_pwd.text()
self.cur.execute(''' SELECT add_username FROM users=add_username''')
data = self.cur.fetchall()
self.cur.execute(''' INSERT INTO users(add_fullname,
add_username, add_Email, add_password, add_phone,
add_birthday, add_profession, add_Country, add_keyone, add_keytwo, add_newpassword)
VALUE (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s ) ''',
(add_fullname, add_username, add_Email, add_password, add_phone, add_birthday,
add_profession, add_Country, add_keyone, add_keytwo, add_newpassword))
self.mysql_db.commit()
self.line_fullname.clear()
self.line_username.clear()
self.line_email.clear()
self.line_password.clear()
this code for fill profile :
def save_profil(self):
self.tabWidget.setCurrentIndex(6)
add_fullname = self.line_fullname.text()
add_Email = self.line_email.text()
add_phone = self.line_telephon.text()
add_birthday = self.dateofbirth.text()
add_profession = self.line_profession.text()
add_Country = self.line_country.text()
add_keyone = self.keyone.text()
add_keytwo = self.keytwo.text()
add_newpassword = self.new_pwd.text()
self.cur.execute(''' INSERT INTO users(add_fullname, add_Email, add_phone, add_birthday, add_profession,
add_Country, add_keyone, add_keytwo, add_newpassword)
VALUE (%s, %s, %s, %s, %s, %s, %s, %s, %s ) ''',
(add_fullname, add_Email,
add_phone, add_birthday, add_profession,
add_Country, add_keyone, add_keytwo, add_newpassword))
self.mysql_db.commit()
print("profil edited")
You should use UPDATE instead of INSERT in the "save_profil" function if the profile already exists.
UPDATE table_name
SET column1 = value1, column2 = value2, ...
WHERE condition;
You will also need to indicate which user should be updated in
WHERE condition
I would also advise you to read the user ID or another unique key you have defined in your table. So you can update the correct user.
Ex:
def save_profil(self):
self.tabWidget.setCurrentIndex(6)
add_fullname = self.line_fullname.text()
add_Email = self.line_email.text()
add_phone = self.line_telephon.text()
add_birthday = self.dateofbirth.text()
add_profession = self.line_profession.text()
add_Country = self.line_country.text()
add_keyone = self.keyone.text()
add_keytwo = self.keytwo.text()
add_newpassword = self.new_pwd.text()
query = "UPDATE users SET add_fullname=%s, add_Email=%s, add_phone=%s, add_birthday=%s, add_profession=%s, add_Country=%s, add_keyone=%s, add_keytwo=%s, add_newpassword= %s WHERE (UNIQUE_KEY = %s)"
data = (add_fullname, add_Email,add_phone, add_birthday, add_profession,add_Country, add_keyone, add_keytwo, add_newpassword, UNIQUE_KEY_VALUE)
self.cur.execute(query,data)
self.mysql_db.commit()
print("profil edited")
Note that UNIQUE_KEY and UNIQUE_KEY_VALUE are suggestions, you must define these values according to your model.
As I can see in the figure that shows the database, you can use the "id" to know which user should be updated.For this case, UNIQUE_KEY would be replaced by id, and UNIQUE_KEY_VALUE would be the user id that needs to be updated, in your example it would be 14, but you need to get that value inside your function, just like you got the other values.
I am getting some JSON data from a third party API. I am trying to add that data into my own database to be used for a website. I loop through each record in the JSON and execute a SQL query to insert that data into my database. However some records in the JSON data doesn't exist, and therefore causes my query to fail. I have set defaults for these fields for this reason however it still falls over.
isNonFoilOnly field will only appear in some of of the records in the JSON data.
models.py
class Set(models.Model):
code = models.CharField(max_length=100, unique=True)
keyrune_code = models.CharField(max_length=100)
name = models.CharField(max_length=100)
type = models.CharField(max_length=100)
release_date = models.DateField()
base_set_size = models.IntegerField()
total_set_size = models.IntegerField()
is_online_only = models.BooleanField(default=False)
is_non_foil_only = models.BooleanField(default=False)
is_foil_only = models.BooleanField(default=False)
sale_status = models.BooleanField(default=False)
def __str__(self):
return self.name
views.py
response = requests.request("GET", "https://mtgjson.com/api/v5/SetList.json")
data = response.json()["data"]
sorted_obj = sorted(data, key=lambda k: k['releaseDate'], reverse=False)
sql = """
INSERT INTO dashboard_set
(code, keyrune_code, name, type, release_date, base_set_size, total_set_size, is_online_only, is_non_foil_only, is_foil_only, sale_status)
VALUES
( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s )
ON CONFLICT (code) DO UPDATE
SET keyrune_code = %s,
name = %s,
type = %s,
release_date = %s,
base_set_size = %s,
total_set_size = %s,
is_online_only = %s,
is_non_foil_only = %s,
is_foil_only = %s;
"""
conn = None
try:
params = config()
conn = psycopg2.connect(**params)
cur = conn.cursor()
for entry in sorted_obj:
cur.execute(sql, (
entry["code"],
entry["keyruneCode"],
entry["name"],
entry["type"],
entry["releaseDate"],
entry["baseSetSize"],
entry["totalSetSize"],
entry["isOnlineOnly"],
entry["isNonFoilOnly"],
entry["isFoilOnly"],
False,
entry["keyruneCode"],
entry["name"],
entry["type"],
entry["releaseDate"],
entry["baseSetSize"],
entry["totalSetSize"],
entry["isOnlineOnly"],
entry["isNonFoilOnly"],
entry["isFoilOnly"]
))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return redirect('dashboard:sets')
You seem to using Django and not using it at the same time. The Django way to do this is:
from yourapp.models import Set
def yourview(request):
response = requests.request("GET", "https://mtgjson.com/api/v5/SetList.json")
data = response.json()["data"]
# Not caring about sort, because why?
for entry in data:
code = data.pop('code', None)
if not code:
continue # or raise
Set.objects.update_or_create(code=code, defaults=data)
return redirect('dashboard:sets')
I'm trying to put all information from excel to mysql, while processing it have these problem.
struggling to solve it!
counted all %s, seems like didn't miss any of them.
query = """INSERT INTO sanction (id, organization_type, organization, date, decision_number, penalty_type, penalty_way
penalty, violation, execution_period, article, note, type_npa, department, uploaded_date)
VALUES(null, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"""
for r in range(1, sheet.nrows):
organization_type = sheet.cell(r,1).value
organization = sheet.cell(r,2).value
date = sheet.cell(r,3).value
decision_number = sheet.cell(r,4).value
penalty_type = sheet.cell(r,5).value
penalty_way = sheet.cell(r,6).value
penalty = sheet.cell(r,7).value
violation = sheet.cell(r,8).value
execution_period = sheet.cell(r,9).value
article =sheet.cell(r,10).value
note =sheet.cell(r,11).value
type_npa =sheet.cell(r,12).value
department =sheet.cell(r,13).value
uploaded_date =datetime.now().strftime("%Y-%m-%d %H:%M")
values = (organization_type, organization, date, decision_number, penalty_type,
penalty_way,penalty, violation, execution_period,article, note, type_npa, department,uploaded_date)
mycursor.execute(query, [values])
I notice 2 things that could cause this error:
Your variable values is a tuple already, so you dont need to wrap it inside a new list.
That means, change this line
mycursor.execute(query, [values])
to
mycursor.execute(query, values)
You are also missing a comma in your query in the part where you list the target column names, between penalty_way and penalty.
In case of this many arguments, I would suggest to restructure your code so that you can more easily see if you missed anything.
For example, here is a version that groups the 15 parameters in a 1-3-3-3-3-2 formation in 3 parts: the first part of the query, the second part of the query and also when building the values tuple.
query = """
INSERT INTO sanction (
id,
organization_type, organization, date,
decision_number, penalty_type, penalty_way,
penalty, violation, execution_period,
article, note, type_npa,
department, uploaded_date)
VALUES (
null,
%s, %s, %s,
%s, %s, %s,
%s, %s, %s,
%s, %s, %s,
%s, %s)
"""
for r in range(1, sheet.nrows):
organization_type = sheet.cell(r, 1).value
organization = sheet.cell(r, 2).value
date = sheet.cell(r, 3).value
decision_number = sheet.cell(r, 4).value
penalty_type = sheet.cell(r, 5).value
penalty_way = sheet.cell(r, 6).value
penalty = sheet.cell(r, 7).value
violation = sheet.cell(r, 8).value
execution_period = sheet.cell(r, 9).value
article = sheet.cell(r, 10).value
note = sheet.cell(r, 11).value
type_npa = sheet.cell(r, 12).value
department = sheet.cell(r, 13).value
uploaded_date = datetime.now().strftime("%Y-%m-%d %H:%M")
values = (
# the first value of the INSERT statement will be NULL
organization_type, organization, date, # 3 elements
decision_number, penalty_type, penalty_way, # 3 elements
penalty, violation, execution_period, # 3 elements
article, note, type_npa, # 3 elements
department, uploaded_date, # 2 elements
)
mycursor.execute(query, values)
I have a table and I want to translate columns 'topic' and 'review' of a row and store the entire table with their translations into a new table. It seems that the for-loop doesn't iterate over all rows of the input table. Only the first row is stored into the new table. Why?
database = mysql.connector.connect(user='root', password='root', host='localhost', database='test')
DBcursor = database.cursor(buffered=True)
query = ("SELECT * FROM test_de")
DBcursor.execute(query)
for (id, user_name, date, country, version, score, topic, review, url) in DBcursor:
topic_trans = translate(topic, 'en')
review_trans = translate(review, 'en')
add_translation = ("INSERT INTO test_de_en(id, user_name, date, country, version, score, topic, review, url)"
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)")
translation_data = (id, user_name, date, country, version, score, topic_trans, review_trans, url)
DBcursor.execute(add_translation, translation_data)
database.commit()
DBcursor.close()
database.close()
I have an input field that is basically a comma delimited string (i.e. something like "deniscm, toms, peters"). That information is sent via AJAX to my Python handler SaveQueryPage. What I want to do is parse this information as a list and then insert each entry into my database. My code is as follows, but it doesn't work unfortunately. Any suggestions?
Python code:
class SaveQueryPage(webapp2.RequestHandler):
def post(self):
user = users.get_current_user()
user_nickname = user.nickname()
query_name = self.request.get('queryName')
query_collab = self.request.get('queryCollab')
query_collaborators = re.split(r'\s*[,]\s*', query_collab.strip())
query_collaborators = query_collaborators.append(user_nickname)
query_collaborators = filter(None, query_collaborators)
conn = rdbms.connect(instance=_INSTANCE_NAME, database='queryInfo')
cursor = conn.cursor()
cursor.execute('INSERT INTO queries (userNickname, queryName) VALUES (%s, %s)', (user_nickname, query_name))
conn.commit()
for item in query_collaborators:
cursor = conn.cursor()
cursor.execute('INSERT INTO collaborators (queryName, userNickname) VALUES (%s, %s)', (query_name, item))
conn.commit()
conn.close()
I finally managed to get it working. Looks like the regular expression turned the items in the list to a unicode format, which was only caught when I added some logs. I also had an error in appending a string to the list. Thanks for the pointers! The code below now works for me:
class SaveQueryPage(webapp2.RequestHandler):
def post(self):
user = users.get_current_user()
user_nickname = user.nickname()
user_email = user.email()
query_name = self.request.get('queryName')
query_description = self.request.get('queryDescription')
query_collab = self.request.get('queryCollab')
logging.info('Data read for query_collab is %s', query_collab)
query_collab_re = re.split(r'\s*[,;]\s*', query_collab.strip())
logging.info('Data read for query_collab_re is %s', query_collab_re)
query_collab_decode = []
for item in query_collab_re:
item = str(item)
query_collab_decode.append(item)
logging.info('Data read for query_collab_decode is %s', query_collab_decode)
query_collab_decode.append(user_nickname)
logging.info('Data read for query_collab_append is %s', query_collab_decode)
query_collab_filter = filter(None, query_collab_decode)
logging.info('Data read for query_collab_filter is %s', query_collab_filter)
query_value = self.request.get('queryValue') # query_value
date_created = datetime.today()
date_lastupdated = datetime.today()
active_flag = "true"
random_id = random.randint(1000000000000, 9999999999999)
unique_query_id = user_nickname + "_" + str(random_id)
conn = rdbms.connect(instance=_INSTANCE_NAME, database='userPrefs')
cursor = conn.cursor()
cursor.execute('INSERT INTO queries (userNickname, queryName, queryDescription, queryValue, dateCreated, dateLastUpdated, activeFlag, uniqueId) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)', (user_nickname, query_name, query_description, query_value, date_created, date_lastupdated, active_flag, unique_query_id))
conn.commit()
try:
for item in query_collab_filter:
cursor = conn.cursor()
cursor.execute('INSERT INTO collaborators (uniqueId, userNickname) VALUES (%s, %s)', (unique_query_id, item))
conn.commit()
except:
logging.error('There was an error inserting the values into the collaborators table. query_collaborators =' + str(query_collaborators))
conn.close()