I'm trying to put all information from excel to mysql, while processing it have these problem.
struggling to solve it!
counted all %s, seems like didn't miss any of them.
query = """INSERT INTO sanction (id, organization_type, organization, date, decision_number, penalty_type, penalty_way
penalty, violation, execution_period, article, note, type_npa, department, uploaded_date)
VALUES(null, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"""
for r in range(1, sheet.nrows):
organization_type = sheet.cell(r,1).value
organization = sheet.cell(r,2).value
date = sheet.cell(r,3).value
decision_number = sheet.cell(r,4).value
penalty_type = sheet.cell(r,5).value
penalty_way = sheet.cell(r,6).value
penalty = sheet.cell(r,7).value
violation = sheet.cell(r,8).value
execution_period = sheet.cell(r,9).value
article =sheet.cell(r,10).value
note =sheet.cell(r,11).value
type_npa =sheet.cell(r,12).value
department =sheet.cell(r,13).value
uploaded_date =datetime.now().strftime("%Y-%m-%d %H:%M")
values = (organization_type, organization, date, decision_number, penalty_type,
penalty_way,penalty, violation, execution_period,article, note, type_npa, department,uploaded_date)
mycursor.execute(query, [values])
I notice 2 things that could cause this error:
Your variable values is a tuple already, so you dont need to wrap it inside a new list.
That means, change this line
mycursor.execute(query, [values])
to
mycursor.execute(query, values)
You are also missing a comma in your query in the part where you list the target column names, between penalty_way and penalty.
In case of this many arguments, I would suggest to restructure your code so that you can more easily see if you missed anything.
For example, here is a version that groups the 15 parameters in a 1-3-3-3-3-2 formation in 3 parts: the first part of the query, the second part of the query and also when building the values tuple.
query = """
INSERT INTO sanction (
id,
organization_type, organization, date,
decision_number, penalty_type, penalty_way,
penalty, violation, execution_period,
article, note, type_npa,
department, uploaded_date)
VALUES (
null,
%s, %s, %s,
%s, %s, %s,
%s, %s, %s,
%s, %s, %s,
%s, %s)
"""
for r in range(1, sheet.nrows):
organization_type = sheet.cell(r, 1).value
organization = sheet.cell(r, 2).value
date = sheet.cell(r, 3).value
decision_number = sheet.cell(r, 4).value
penalty_type = sheet.cell(r, 5).value
penalty_way = sheet.cell(r, 6).value
penalty = sheet.cell(r, 7).value
violation = sheet.cell(r, 8).value
execution_period = sheet.cell(r, 9).value
article = sheet.cell(r, 10).value
note = sheet.cell(r, 11).value
type_npa = sheet.cell(r, 12).value
department = sheet.cell(r, 13).value
uploaded_date = datetime.now().strftime("%Y-%m-%d %H:%M")
values = (
# the first value of the INSERT statement will be NULL
organization_type, organization, date, # 3 elements
decision_number, penalty_type, penalty_way, # 3 elements
penalty, violation, execution_period, # 3 elements
article, note, type_npa, # 3 elements
department, uploaded_date, # 2 elements
)
mycursor.execute(query, values)
Related
A new member has been registered in the database, but when entering the profile modification page and adding other information, it is registered in a new row and not in the same information as the registered member.
register new member :
def add_users(self):
add_fullname = self.line_fullname.text()
add_username = self.line_username.text()
add_Email = self.line_email.text()
add_password = self.line_password.text()
add_phone = self.line_telephon.text()
add_birthday = self.dateofbirth.text()
add_profession = self.line_profession.text()
add_Country = self.line_country.text()
add_keyone = self.keyone.text()
add_keytwo = self.keytwo.text()
add_newpassword = self.new_pwd.text()
self.cur.execute(''' SELECT add_username FROM users=add_username''')
data = self.cur.fetchall()
self.cur.execute(''' INSERT INTO users(add_fullname,
add_username, add_Email, add_password, add_phone,
add_birthday, add_profession, add_Country, add_keyone, add_keytwo, add_newpassword)
VALUE (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s ) ''',
(add_fullname, add_username, add_Email, add_password, add_phone, add_birthday,
add_profession, add_Country, add_keyone, add_keytwo, add_newpassword))
self.mysql_db.commit()
self.line_fullname.clear()
self.line_username.clear()
self.line_email.clear()
self.line_password.clear()
this code for fill profile :
def save_profil(self):
self.tabWidget.setCurrentIndex(6)
add_fullname = self.line_fullname.text()
add_Email = self.line_email.text()
add_phone = self.line_telephon.text()
add_birthday = self.dateofbirth.text()
add_profession = self.line_profession.text()
add_Country = self.line_country.text()
add_keyone = self.keyone.text()
add_keytwo = self.keytwo.text()
add_newpassword = self.new_pwd.text()
self.cur.execute(''' INSERT INTO users(add_fullname, add_Email, add_phone, add_birthday, add_profession,
add_Country, add_keyone, add_keytwo, add_newpassword)
VALUE (%s, %s, %s, %s, %s, %s, %s, %s, %s ) ''',
(add_fullname, add_Email,
add_phone, add_birthday, add_profession,
add_Country, add_keyone, add_keytwo, add_newpassword))
self.mysql_db.commit()
print("profil edited")
You should use UPDATE instead of INSERT in the "save_profil" function if the profile already exists.
UPDATE table_name
SET column1 = value1, column2 = value2, ...
WHERE condition;
You will also need to indicate which user should be updated in
WHERE condition
I would also advise you to read the user ID or another unique key you have defined in your table. So you can update the correct user.
Ex:
def save_profil(self):
self.tabWidget.setCurrentIndex(6)
add_fullname = self.line_fullname.text()
add_Email = self.line_email.text()
add_phone = self.line_telephon.text()
add_birthday = self.dateofbirth.text()
add_profession = self.line_profession.text()
add_Country = self.line_country.text()
add_keyone = self.keyone.text()
add_keytwo = self.keytwo.text()
add_newpassword = self.new_pwd.text()
query = "UPDATE users SET add_fullname=%s, add_Email=%s, add_phone=%s, add_birthday=%s, add_profession=%s, add_Country=%s, add_keyone=%s, add_keytwo=%s, add_newpassword= %s WHERE (UNIQUE_KEY = %s)"
data = (add_fullname, add_Email,add_phone, add_birthday, add_profession,add_Country, add_keyone, add_keytwo, add_newpassword, UNIQUE_KEY_VALUE)
self.cur.execute(query,data)
self.mysql_db.commit()
print("profil edited")
Note that UNIQUE_KEY and UNIQUE_KEY_VALUE are suggestions, you must define these values according to your model.
As I can see in the figure that shows the database, you can use the "id" to know which user should be updated.For this case, UNIQUE_KEY would be replaced by id, and UNIQUE_KEY_VALUE would be the user id that needs to be updated, in your example it would be 14, but you need to get that value inside your function, just like you got the other values.
I am trying to insert multiple rows from dataframe to mysql.
I counted arguments numbers and they all are of same number (i.e. 11 here).
But still I get this
not enough arguments for format string
Here's the function:
def insert_result_sets_into_db(df, filter_rule_id):
cols = "('hash_id', 'filter_rule_id', 'task_id', 'assigned_to', 'human_verdict', 'verdict_date', 'verdict_by', 'created_by', 'updated_by', 'created_at', 'updated_at')"
if not df.empty:
with connections['frontend'].cursor() as cursor:
sql = "INSERT INTO dmf_result_set_assign " + cols + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
values = []
now = datetime.now()
formatted_date = now.strftime('%Y-%m-%d %H:%M:%S')
# Insert DataFrame records one by one.
for i,row in df.iterrows():
tup = (row['_id'], filter_rule_id, 1, 1, 'Matched', formatted_date, 1, 1, 1, formatted_date, formatted_date)
values.extend(tup)
cursor.executemany(sql, values)
connections.commit()
Am I missing anything here?
Main idea of these work is to parse each page from pagination(it has 152 pages), and put these data to db. while running error occurred, column 'execution_period' cannot be null".
in my DataFrame I did this:
df['Срок исполнения'] = df['Срок исполнения'].fillna(0)
df['Дата принятия решения/дела'] = df['Дата принятия решения/дела'].fillna(0)
don't know how to fix it.
df = pd.DataFrame()
df_list = pd.read_html('sanction.xls')
df = df_list[0]
database = MySQLdb.connect (host="host", user = "root", passwd = "password", db = "db", charset='utf8')
mycursor = database.cursor()
cursor = database.cursor()
query = """INSERT INTO sanction (
id,
organization_type, organization, date,
decision_number, penalty_type, penalty_way,
penalty, violation, execution_period,
article, note, type_npa,
department, uploaded_date)
VALUES (
null,
%s, %s, STR_TO_DATE(%s,"%%d.%%m.%%Y"),
%s, %s, %s,
%s, %s, STR_TO_DATE(%s,"%%d.%%m.%%Y"),
%s, %s, %s,
%s, %s)"""
for r in range(1, sheet.nrows):
organization_type = sheet.cell(r,1).value
organization = sheet.cell(r,2).value
date = (sheet.cell(r,3).value)
decision_number = sheet.cell(r,4).value
penalty_type = sheet.cell(r,5).value
penalty_way = sheet.cell(r,6).value
penalty = sheet.cell(r,7).value
violation = sheet.cell(r,8).value
execution_period = sheet.cell(r,9).value
article =sheet.cell(r,10).value
note =sheet.cell(r,11).value
type_npa =sheet.cell(r,12).value
department =sheet.cell(r,13).value
uploaded_date =datetime.now().strftime("%Y-%m-%d %H:%M")
values = (organization_type, organization, date, decision_number, penalty_type,
penalty_way,penalty, violation, execution_period,article, note, type_npa, department,uploaded_date)
mycursor.execute(query, values)
I have thousands of related CSVs and I want to write their contents to a Postgres table in a way that includes metadata about where each row came from.
I am not clear on how to write the variables I created near the top of my script into the table.
Can anyone advise?
target_directory = Path(sys.argv[1]).resolve()
# FOR THE WAC AND RAC DATASETS
for file in target_directory.rglob('*.csv'):
print(str(file.stem).split('_'))
state = str(file.stem).split('_')[0]
data_category = str(file.stem).split('_')[1]
workforce_segment = str(file.stem).split('_')[2] # THIS IS DIFFERENT FROM THE O-D DATASETS
job_type = str(file.stem).split('_')[3]
year = str(file.stem).split('_')[4]
print('Writing: ' + str(file.name))
# MAKE SURE THIS IS THE RIGHT TABLE FOR THE FILES
cur.execute(create_table_WAC)
with open(file,'r') as file_in:
# INSERT THE DATA IN USING THE COLUMN NAMES....SO YOU CAN ADD YOUR SPLIT STRING INFO ABOVE.....
# MAKE SURE THIS HAS THE RIGHT TABLE NAME IN THE COPY STATEMENT
cur.execute("INSERT INTO opendata_uscensus_usa_lodes_wac (serial_id, state_name, data_category, workforce_segment, job_type, year, w_geocode, C000, CA01, CA02, CA03, CE01, CE02) \
VALUES (%s, state_name, data_category, workforce_segment, job_type, year, %s, %s, %s, %s, %s, %s)")
conn.commit()
conn.close()
As per PEP-249 (Python Database API Specification) which most DB-APIs adhere to including pymssql, cx_oracle, ibm_db, pymysql, sqlite3, and pyodbc, in psycopg2 variables to be binded as parameters in prepared statements would go into the second argument of cur.execute(query, params).
Specifically, combine your file level variables with CSV variables during iteration and pass them as a list or tuple of parameters into execution call. Below uses the csv.DictReader method that builds a dictionary of every row from csv data.
NOTE: below query leaves out primary key, serial_id, which should populate via a sequence in Postgres table.
for file in target_directory.rglob('*.csv'):
print(str(file.stem).split('_'))
# FILE LEVEL VARIABLES
state_name = str(file.stem).split('_')[0]
data_category = str(file.stem).split('_')[1]
workforce_segment = str(file.stem).split('_')[2]
job_type = str(file.stem).split('_')[3]
year = str(file.stem).split('_')[4]
# PREPARED STATEMENT
sql = """INSERT INTO opendata_uscensus_usa_lodes_wac
(state_name, data_category, workforce_segment,
job_type, year, w_geocode, C000, CA01, CA02, CA03, CE01, CE02)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"""
with open(file,'r') as file_in:
# ITERATE THROUGH FOR CSV VARIABLES
reader = csv.DictReader(file_in)
for row in reader:
cur.execute(sql, (state,data_category,workforce_segment,job_type,year,
row['w_geocode'], row['C000'], row['CA01'],
row['CA02'], row['CA03'], row['CE01'], row['CE02'])
)
conn.commit()
I have a table and I want to translate columns 'topic' and 'review' of a row and store the entire table with their translations into a new table. It seems that the for-loop doesn't iterate over all rows of the input table. Only the first row is stored into the new table. Why?
database = mysql.connector.connect(user='root', password='root', host='localhost', database='test')
DBcursor = database.cursor(buffered=True)
query = ("SELECT * FROM test_de")
DBcursor.execute(query)
for (id, user_name, date, country, version, score, topic, review, url) in DBcursor:
topic_trans = translate(topic, 'en')
review_trans = translate(review, 'en')
add_translation = ("INSERT INTO test_de_en(id, user_name, date, country, version, score, topic, review, url)"
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)")
translation_data = (id, user_name, date, country, version, score, topic_trans, review_trans, url)
DBcursor.execute(add_translation, translation_data)
database.commit()
DBcursor.close()
database.close()