Ive been trying to sort out how to import data into a sqlite table, I can do this but I seem to have an error issue with the date entries.
Here is reproducible code that demonstrates the error
The .txt file
1,2019-08-24
2,2019-08-24
and the .py file
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from datetime import datetime
import sqlite3
import importlib
import subprocess
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///mydatabase.db'
db = SQLAlchemy(app)
class MyTable(db.Model):
__tablename__ = 'myTable'
id = db.Column(db.Integer, primary_key=True)
date_created = db.Column(db.DateTime)
db.create_all()
p = subprocess.Popen(["sqlite3", "mydatabase.db"], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
p.communicate(b"""
INSERT INTO mytable (id, date_created);
.separator ","
.import repro.txt mytable """)
rows = MyTable.query.all()
for row in rows:
mytable_update = MyTable.query.get_or_404(row.id)
mytable_update.date_created = datetime.strptime(mytable_update.date_created, "%Y-%m-%d").date()
db.session.commit()
which gives the error
ValueError: Couldn't parse datetime string: '2019-08-24 '
or the full error message I get is
(env) (base) Benjamats-Air:helloPython benjamattesjaroen$ python repro.py
/Users/benjamattesjaroen/helloPython/env/lib/python3.7/site-packages/flask_sqlalchemy/__init__.py:835: FSADeprecationWarning: SQLALCHEMY_TRACK_MODIFICATIONS adds significant overhead and will be disabled by default in the future. Set it to True or False to suppress this warning.
'SQLALCHEMY_TRACK_MODIFICATIONS adds significant overhead and '
Error: near line 2: near ";": syntax error
Traceback (most recent call last):
File "repro.py", line 26, in <module>
rows = MyTable.query.all()
File "/Users/benjamattesjaroen/helloPython/env/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3178, in all
return list(self)
File "/Users/benjamattesjaroen/helloPython/env/lib/python3.7/site-packages/sqlalchemy/orm/loading.py", line 105, in instances
util.raise_from_cause(err)
File "/Users/benjamattesjaroen/helloPython/env/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/Users/benjamattesjaroen/helloPython/env/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 153, in reraise
raise value
File "/Users/benjamattesjaroen/helloPython/env/lib/python3.7/site-packages/sqlalchemy/orm/loading.py", line 85, in instances
rows = [proc(row) for row in fetch]
File "/Users/benjamattesjaroen/helloPython/env/lib/python3.7/site-packages/sqlalchemy/orm/loading.py", line 85, in <listcomp>
rows = [proc(row) for row in fetch]
File "/Users/benjamattesjaroen/helloPython/env/lib/python3.7/site-packages/sqlalchemy/orm/loading.py", line 572, in _instance
populators,
File "/Users/benjamattesjaroen/helloPython/env/lib/python3.7/site-packages/sqlalchemy/orm/loading.py", line 693, in _populate_full
dict_[key] = getter(row)
File "/Users/benjamattesjaroen/helloPython/env/lib/python3.7/site-packages/sqlalchemy/engine/result.py", line 107, in __getitem__
return processor(self._row[index])
File "/Users/benjamattesjaroen/helloPython/env/lib/python3.7/site-packages/sqlalchemy/processors.py", line 43, in process
"'%s'" % (type_.__name__, value)
ValueError: Couldn't parse datetime string: '2019-08-24 '
The .txt file seems to contain entries of type Date, and not Datetime. You could convert the data for the following rows into type Datetime as specified by sqlalchemy: Date - datetime description.
If all of your data in that column contains '2019-08-24' or 'YYYY-MM-DD' format then you can safely change your database column type to Column(Date).
Related
I am learning sqlalchemy core with postgresql database in python.
I tried to run the following script and got this error message:
from sqlalchemy import create_engine
from sqlalchemy import Table, MetaData, String
engine = create_engine('postgresql://postgres:123456#localhost:5432/red30')
with engine.connect() as connection:
meta = MetaData(engine)
sales_table = Table('sales', meta)
# Create
insert_statement = sales_table.insert().values(order_num=1105911,
order_type='Retail',
cust_name='Syman Mapstone',
prod_number='EB521',
prod_name='Understanding Artificial Intelligence',
quantity=3,
price=19.5,
discount=0,
order_total=58.5)
connection.execute(insert_statement)
# Read
select_statement = sales_table.select().limit(10)
result_set = connection.execute(select_statement)
for r in result_set:
print(r)
# Update
update_statement = sales_table.update().where(sales_table.c.order_num==1105910).values(quantity=2, order_total=39)
connection.execute(update_statement)
# Confirm Update: Read
reselect_statement = sales_table.select().where(sales_table.c.order_num==1105910)
updated_set = connection.execute(reselect_statement)
for u in updated_set:
print(u)
# Delete
delete_statement = sales_table.delete().where(sales_table.c.order_num==1105910)
connection.execute(delete_statement)
# Confirm Delete: Read
not_found_set = connection.execute(reselect_statement)
print(not_found_set.rowcount)
error message:
(postgres-prac) E:\xfile\postgresql\postgres-prac>python postgres-sqlalchemy-core.py
Traceback (most recent call last):
File "postgres-sqlalchemy-core.py", line 20, in <module>
connection.execute(insert_statement)
File "E:\xfile\postgresql\postgres-prac\lib\site-packages\sqlalchemy\engine\ba
se.py", line 1414, in execute
return meth(
File "E:\xfile\postgresql\postgres-prac\lib\site-packages\sqlalchemy\sql\eleme
nts.py", line 485, in _execute_on_connection
return connection._execute_clauseelement(
File "E:\xfile\postgresql\postgres-prac\lib\site-packages\sqlalchemy\engine\ba
se.py", line 1630, in _execute_clauseelement
compiled_sql, extracted_params, cache_hit = elem._compile_w_cache(
File "E:\xfile\postgresql\postgres-prac\lib\site-packages\sqlalchemy\sql\eleme
nts.py", line 651, in _compile_w_cache
compiled_sql = self._compiler(
File "E:\xfile\postgresql\postgres-prac\lib\site-packages\sqlalchemy\sql\eleme
nts.py", line 290, in _compiler
return dialect.statement_compiler(dialect, self, **kw)
File "E:\xfile\postgresql\postgres-prac\lib\site-packages\sqlalchemy\sql\compi
ler.py", line 1269, in __init__
Compiled.__init__(self, dialect, statement, **kwargs)
File "E:\xfile\postgresql\postgres-prac\lib\site-packages\sqlalchemy\sql\compi
ler.py", line 710, in __init__
self.string = self.process(self.statement, **compile_kwargs)
File "E:\xfile\postgresql\postgres-prac\lib\site-packages\sqlalchemy\sql\compi
ler.py", line 755, in process
return obj._compiler_dispatch(self, **kwargs)
File "E:\xfile\postgresql\postgres-prac\lib\site-packages\sqlalchemy\sql\visit
ors.py", line 143, in _compiler_dispatch
return meth(self, **kw) # type: ignore # noqa: E501
File "E:\xfile\postgresql\postgres-prac\lib\site-packages\sqlalchemy\sql\compi
ler.py", line 5317, in visit_insert
crud_params_struct = crud._get_crud_params(
File "E:\xfile\postgresql\postgres-prac\lib\site-packages\sqlalchemy\sql\crud.
py", line 326, in _get_crud_params
raise exc.CompileError(
sqlalchemy.exc.CompileError: Unconsumed column names: order_type, quantity, cust
_name, discount, prod_number, price, order_total, order_num, prod_name
You define your table as an empty table:
sales_table = Table('sales', meta)
So when trying to insert a record with all those keywords, they cannot be mapped to columns and do not get consumed, hence the Unconsumed column names error.
You need to define the table columns in your Table creation. See the following example from the docs:
from sqlalchemy import Table, Column, Integer, String
user = Table(
"user",
metadata_obj,
Column("user_id", Integer, primary_key=True),
Column("user_name", String(16), nullable=False),
Column("email_address", String(60)),
Column("nickname", String(50), nullable=False),
)
I am using pyfirebirdsql version 0.8.5 to connect to Interbase database.
import firebirdsql
conn = firebirdsql.connect(
host='192.168.133.121',
database='E:\\test\test.gdb',
port=3050,
user='sysdba',
password='masterkey'
#charset="WIN1251"
#charset="UTF8"
)
cur = conn.cursor()
cur.execute("select column1 from table1")
for c in cur.fetchall():
print(c)
conn.close()
When I try to run the script I get the following:
Traceback (most recent call last):
File "123.py", line 15, in <module>
for c in cur.fetchall():
File "/usr/lib/python2.7/site-packages/firebirdsql/fbcore.py", line 291, in fetchall
return [tuple(r) for r in self._fetch_records]
File "/usr/lib/python2.7/site-packages/firebirdsql/fbcore.py", line 225, in _fetch_generator
stmt_handle, self._xsqlda)
File "/usr/lib/python2.7/site-packages/firebirdsql/wireprotocol.py", line 728, in _op_fetch_response
return self._parse_op_response() # error occured
File "/usr/lib/python2.7/site-packages/firebirdsql/wireprotocol.py", line 210, in _parse_op_response
raise OperationalError(message, gds_codes, sql_code)
firebirdsql.OperationalError: arithmetic exception, numeric overflow, or string truncation
Cannot transliterate character between character sets
Adding charset both "WIN1251" and "UTF8" doesn't solve the issue. Do you have any ideas?
python program ,insert data in mysql table:
is_exist_table_sql = "SHOW TABLES LIKE 'fb_public_figure_posts'"
if cur.execute(is_exist_table_sql) == 0:
create_sql = """CREATE TABLE fb_public_figure_posts(Post_ID varchar(32), Permalink varchar(128), Create_time varchar(32), Updated_time varchar(32), Author varchar(32),
Author_ID bigint, Message text, Link varchar(1024), Likes int, Comments int, pf_ID bigint, foreign key(pf_ID) references fb_public_figure_info(ID))"""
cur.execute(create_sql)
db_data = posts
if type == "public_figure_posts":
for item in db_data:
if "'" in item["message"]:
item["message"] = str(item["message"]).replace("'","\\\'")
elif '"' in item["message"]:
item["message"] = str(item["message"]).replace('"','\\\"')
is_exist_id_sql = "select * from fb_public_figure_posts where Post_ID = '" + item['id'] + "'"
if cur.execute(is_exist_id_sql) == 0:
insert_sql = "INSERT INTO fb_public_figure_posts VALUES ('{0}','{1}','{2}','{3}','{4}',{5},'{6}','{7}',{8},{9},{10})".format(item['id'],item['permalink_url'],item['created_time'],item['updated_time'],item['from']['name'],item['from']['id'],item['message'],item['link'],item['likes']['summary']['total_count'],item['comments']['summary']['total_count'],public_figure_id)
print(insert_sql)
cur.execute(insert_sql)
when running , it errors:
pymysql.err.InternalError: (1366, "Incorrect string value: '\\xF0\\x9F\\x87\\xBA\\xF0\\x9F...' for column 'Message' at row 1")
it point that the sentence error:
Traceback (most recent call last):
File "C:/Python/PyCharmProject/FaceBookCrawl/FBCrawl.py", line 286, in <module>
publicfigure_download.public_figure_posts_storage(public_figure_name)
INSERT INTO fb_public_figure_posts VALUES ('153080620724_10158392447835725','https://www.facebook.com/DonaldTrump/posts/10158392447835725:0','2017-01-01T04:59:07+0000','2017-01-23T19:52:49+0000','Donald J. Trump',153080620724,'TO ALL AMERICANS-
File "C:\Python\PyCharmProject\FaceBookCrawl\publicfigure_download.py", line 103, in public_figure_posts_storage
#HappyNewYear & many blessings to you all! Looking forward to a wonderful & prosperous 2017 as we work together to #MAGA🇺🇸','https://www.facebook.com/DonaldTrump/photos/a.488852220724.393301.153080620724/10158392447835725/?type=3',158710,11045,153080620724)
mysql_manage().public_figure_db_manage(type, posts, public_figure_id, public_figure_name)
File "C:\Python\PyCharmProject\FaceBookCrawl\database_manage.py", line 47, in public_figure_db_manage
cur.execute(insert_sql)
File "C:\Python\Python36\lib\site-packages\pymysql\cursors.py", line 166, in execute
result = self._query(query)
File "C:\Python\Python36\lib\site-packages\pymysql\cursors.py", line 322, in _query
conn.query(q)
File "C:\Python\Python36\lib\site-packages\pymysql\connections.py", line 835, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "C:\Python\Python36\lib\site-packages\pymysql\connections.py", line 1019, in _read_query_result
result.read()
File "C:\Python\Python36\lib\site-packages\pymysql\connections.py", line 1302, in read
first_packet = self.connection._read_packet()
File "C:\Python\Python36\lib\site-packages\pymysql\connections.py", line 981, in _read_packet
packet.check_error()
File "C:\Python\Python36\lib\site-packages\pymysql\connections.py", line 393, in check_error
err.raise_mysql_exception(self._data)
File "C:\Python\Python36\lib\site-packages\pymysql\err.py", line 107, in raise_mysql_exception
raise errorclass(errno, errval)
pymysql.err.InternalError: (1366, "Incorrect string value: '\\xF0\\x9F\\x87\\xBA\\xF0\\x9F...' for column 'Message' at row 1")
Process finished with exit code 1
in conclusion, it shows:
pymysql.err.InternalError: (1366, "Incorrect string value: '\\xF0\\x9F\\x87\\xBA\\xF0\\x9F...' for column 'Message' at row 1")
could you please help me for that
You actually have two problems here. The first one is that you're trying to insert a string containing characters that are not supported by your database/table/field encoding - the catch is that MySQL's "utf-8" encoding is NOT really utf8 compliant (surprise, surprise...). The linked SO question gives full explanations, you can also check this for a migration procedure.
Your second issue - which AFAICT isn't responsible for the problem you describe but will cause quite a few other problems anyway - is that you're not using your db connector properly. This makes your code uselessly complicated and quite brittle - proper escaping is a pain as you might have noticed, so better to let the db connector do the job -, but also opens your app to SQL injection attacks. The cure here is simple: use prepared statements. This is actually much simpler than not using them and will kill two birds with one stone.
Guys i kindly need your help to figure out why am getting this error when i ran my script on a server. the server runs a postgres db i think it is something to do with the object type of the data it's been iterated but not sure how to fix that.
Traceback (most recent call last):
File "scripts/change_topup.py", line 58, in <module>
main()
File "scripts/change_topup.py", line 55, in main
process_file(path)
File "scripts/change_topup.py", line 45, in process_file
for enrolment in enrolments:
File "/opt/cv_instances/cv1/autodeploy/branches/nboreports/.venv/lib/python2.7/site-packages/django/db/models/query.py", line 258, in __iter__
self._fetch_all()
File "/opt/cv_instances/cv1/autodeploy/branches/nboreports/.venv/lib/python2.7/site-packages/django/db/models/query.py", line 1074, in _fetch_all
self._result_cache = list(self.iterator())
File "/opt/cv_instances/cv1/autodeploy/branches/nboreports/.venv/lib/python2.7/site-packages/django/db/models/query.py", line 52, in __iter__
results = compiler.execute_sql()
File "/opt/cv_instances/cv1/autodeploy/branches/nboreports/.venv/lib/python2.7/site-packages/django/db/models/sql/compiler.py", line 848, in execute_sql
cursor.execute(sql, params)
File "/opt/cv_instances/cv1/autodeploy/branches/nboreports/.venv/lib/python2.7/site-packages/django/db/backends/utils.py", line 79, in execute
return super(CursorDebugWrapper, self).execute(sql, params)
File "/opt/cv_instances/cv1/autodeploy/branches/nboreports/.venv/lib/python2.7/site-packages/django/db/backends/utils.py", line 64, in execute
return self.cursor.execute(sql, params)
File "/opt/cv_instances/cv1/autodeploy/branches/nboreports/.venv/lib/python2.7/site-packages/django/db/utils.py", line 95, in __exit__
six.reraise(dj_exc_type, dj_exc_value, traceback)
File "/opt/cv_instances/cv1/autodeploy/branches/nboreports/.venv/lib/python2.7/site-packages/django/db/backends/utils.py", line 64, in execute
return self.cursor.execute(sql, params)
django.db.utils.ProgrammingError: operator does not exist: character varying = integer
LINE 1: ...n"."type" = 1 AND "registration_household"."name" IN (SELECT...
^
HINT: No operator matches the given name and argument type(s). You might need to add explicit type casts.
my script is here
# script to update hh top-up locations
import django
django.setup()
import sys
import sys
import xlrd
from django.db import transaction
from maidea.apps.enrolment.models import Enrolment
from maidea.apps.redemption.models import Retailer
from maidea.apps.registration.models import Household
from maidea.apps.interventions.models import Intervention
from maidea.apps.wfp.models import Office
office = Office.objects.get(slug='so-co')
intervention = Intervention.objects.get(project__office=office, slug='cash-for-dadaab-returnees')
print("Using Office: {0}".format(office))
print("Using Intervention: {0}".format(intervention))
def process_file(path):
# get the first worksheet
book = xlrd.open_workbook(path)
first_sheet = book.sheet_by_index(0)
print('Sheet name: {0}'.format(first_sheet.name))
print("Looping through excel rows. Please wait ...")
#counter = 0
with transaction.atomic():
# iterate through rows
for row_idx in range(1, first_sheet.nrows):
# household names
hh_name = str(first_sheet.row_values(row_idx)[0])
# the new top-up locations
new_top_up = str(first_sheet.row_values(row_idx)[2])
households = Household.objects.filter(name__in=hh_name)
top_up = Retailer.objects.filter(name__in=new_top_up)
enrolments = Enrolment.objects.filter(intervention=intervention, household__name__in=households)
#print("Fetched {0} enrolments".format(enrolments.count())
for enrolment in enrolments:
print enrolment
# enrolment.retailer = top_up
# enrolment.save()
#counter += 1
print("{0} enrolments updated".format(counter))
def main():
path = "/opt/cv_instances/cv1/autodeploy/branches/nboreports/maidea/somalia/data_import_files/Enrolment.xlsx"
process_file(path)
print("DONE PROCESSING FILE.")
main()
The file been processed has rows like
HH | top up |
SD803-11H47782 | MERCY US NEW POS TOPUP
SD803-09H03991 | DRC-MOG
Help will highly appreciated.Thanks
The problem is this line
enrolments = Enrolment.objects.filter(intervention=intervention, household__name__in=households):
households is a queryset of Household instances, not names. I think that either of the following would work.
enrolments = Enrolment.objects.filter(intervention=intervention, household__in=hh_name)
enrolments = Enrolment.objects.filter(intervention=intervention, household__name__in=hh_name)
If you use the second one, then it looks like you can remove the households queryset from your code.
I am writing a piece of python code to copy one table from one mysql db to to another mysql db.
I came across some problems like first it was reading null, empty values as 'None' which I had to convert to 'NULL'.
Now it showing following error -
pymysql.err.InternalError: (1630, u"FUNCTION datetime.datetime does not exist.
Check the 'Function Name Parsing and Resolution' section in the Reference Manual")
When I print the row I can see the entry of datetime.datetime(2014, 8, 25, 8, 24, 51).
I tried solve this problem by replacing datetime.datetime by datetime.time (http://pymotw.com/2/datetime/) But that also failed.
My code is as follows :
import re
from db import conn_main ### database from where to copy
from db import conn ### database where to copy
import datetime
curr1 = conn_main.cursor()
curr2 = conn.cursor()
query = 'SELECT * FROM mytable limit 10'
curr1.execute(query)
for row in curr1:
if row[0] is None or not row[0]:
print "error: empty row",row[0]
continue
else:
print "ROW - %s\n" % str(row)
row = re.sub('None','NULL',str(row))
query = 'replace into mytable values ' + str(row)
curr2.execute(query)
curr2.commit()
curr1.close()
curr2.close()
Traceback & output row:
ROW - (1, '501733938','xyz.emails#gmail.com', None, 'https://www.facebook.com/xyz',
None, None, None, None, None, '2014-08-10T06:06:33+0000', None, 'xyz', None,
datetime.datetime(2014, 8, 25, 8, 24, 51), None, None, None, None, None, None, None,
None, None, None, None, None, None, None, None, None)
Traceback (most recent call last):
File "MY_PYTHON_CODE_FILE_PATH", line 390, in <module> curr2.execute(query)
File "/usr/local/lib/python2.7/dist-packages/pymysql/cursors.py", line 132, in execute result = self._query(query)
File "/usr/local/lib/python2.7/dist-packages/pymysql/cursors.py", line 271, in _query conn.query(q)
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 726, in query self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 861, in _read_query_result result.read()
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 1064, in read first_packet = self.connection._read_packet()
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 826, in _read_packet packet.check_error()
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 370, in check_error raise_mysql_exception(self._data)
File "/usr/local/lib/python2.7/dist-packages/pymysql/err.py", line 116, in raise_mysql_exception _check_mysql_exception(errinfo)
File "/usr/local/lib/python2.7/dist-packages/pymysql/err.py", line 112, in _check_mysql_exception raise InternalError(errno, errorvalue)
Can someone help remove this error ... or suggest any other better way to copy table from one database to another in python.
According to mysql manual up to v 5.7 there is neither function DATETIME (even if there is a datetime TYPE) nor package support (datetime.*). I assume it's the python str that generates datetime.datetime from the binary representation of the datetime in your source database
The problem is: you're trying to use python string representation of some_object as SQL. That's wrong.
str(datetime) will look like
datetime.datetime(year, month, day, hour, minute, seconds, milliseconds)
it's not valid sql string.
If you know what this column number is, you can replace it with string value, like this:
row[dt_index] = row[dt_index].isoformat()
or with concrete format your database accepts, e.g:
row[dt_index] = row[dt_index].strftime('%Y-%m-%d %H:%M:%S')
But I suggest use some libraries or parameterised queries.
Building SQL such a way is very bad and unsafe solution.