Develop Reference

How to read Interbase database with Cyrillic text content

I am using pyfirebirdsql version 0.8.5 to connect to Interbase database.
import firebirdsql
conn = firebirdsql.connect(
host='192.168.133.121',
database='E:\\test\test.gdb',
port=3050,
user='sysdba',
password='masterkey'
#charset="WIN1251"
#charset="UTF8"
)
cur = conn.cursor()
cur.execute("select column1 from table1")
for c in cur.fetchall():
print(c)
conn.close()
When I try to run the script I get the following:
Traceback (most recent call last):
File "123.py", line 15, in <module>
for c in cur.fetchall():
File "/usr/lib/python2.7/site-packages/firebirdsql/fbcore.py", line 291, in fetchall
return [tuple(r) for r in self._fetch_records]
File "/usr/lib/python2.7/site-packages/firebirdsql/fbcore.py", line 225, in _fetch_generator
stmt_handle, self._xsqlda)
File "/usr/lib/python2.7/site-packages/firebirdsql/wireprotocol.py", line 728, in _op_fetch_response
return self._parse_op_response() # error occured
File "/usr/lib/python2.7/site-packages/firebirdsql/wireprotocol.py", line 210, in _parse_op_response
raise OperationalError(message, gds_codes, sql_code)
firebirdsql.OperationalError: arithmetic exception, numeric overflow, or string truncation
Cannot transliterate character between character sets
Adding charset both "WIN1251" and "UTF8" doesn't solve the issue. Do you have any ideas?

Select statement in mysql using LIKE (pyqt5)

Newbie here. Id like to ask What could possibly wrong with this code:
'SELECT * FROM A3A_SIS.customer_info WHERE cust_name LIKE %' +self.le_ci_search.text()+ '%'
This line returns an error of this:
TypeError: a bytes-like object is required, not 'tuple'
I am trying to search a column name where theres a word lopez in it.
UPDATE #1:
I use this code as suggested:
def CustSearch(self):
search_text = '%{}%'.format(self.le_ci_search.text())
con = mdb.connect(user='root', passwd='password',
host='localhost', database='A3A_SIS')
with con:
cur = con.cursor()
query = ('SELECT * FROM A3A_SIS.customer_info WHERE cust_name LIKE %s', (search_text))
if cur.execute(query):
QMessageBox.information(self, "Announcement.","Data was found!")
else:
QMessageBox.information(self, "Announcement.","No data was found!")
con.close()
I got this error:
Traceback (most recent call last):
File "/Users/anthonygaupo/Desktop/A3ASIS/A3A_Func.py", line 409, in
CustSearch
if cur.execute(query):
File
"/Users/anthonygaupo/anaconda3/lib/python3.6/site-packages/MySQLdb/cursors.py",
line 250, in execute
self.errorhandler(self, exc, value)
File
"/Users/anthonygaupo/anaconda3/lib/python3.6/site-packages/MySQLdb/connections.py",
line 50, in defaulterrorhandler
raise errorvalue
File
"/Users/anthonygaupo/anaconda3/lib/python3.6/site-packages/MySQLdb/cursors.py",
line 247, in execute
res = self._query(query)
File
"/Users/anthonygaupo/anaconda3/lib/python3.6/site-packages/MySQLdb/cursors.py",
line 411, in _query
rowcount = self._do_query(q)
File
"/Users/anthonygaupo/anaconda3/lib/python3.6/site-packages/MySQLdb/cursors.py",
line 374, in _do_query
db.query(q)
File
"/Users/anthonygaupo/anaconda3/lib/python3.6/site-packages/MySQLdb/connections.py",
line 277, in query
_mysql.connection.query(self, query)
TypeError: a bytes-like object is required, not 'tuple'
I am MYSQL workbench

You would need to put the search text, including the % characters, in quotes.
But you should not do this. Assemble the value outside of the SQL statement and use parameter substitution:
query = '%{}%'.format(self.le_ci_search.text())
cursor.execute('SELECT * FROM A3A_SIS.customer_info WHERE cust_name LIKE %s', (query,))
Edit
You're creating a single tuple and passing it to the cursor as the query. What I said to do is to create a string, and pass that plus the parameter to the cursor:
cur = con.cursor()
query = 'SELECT * FROM A3A_SIS.customer_info WHERE cust_name LIKE %s'
if cur.execute(query, (search_text,)):
...

python:mysql error, "Incorrect string value: '\\xF0\\x9F\\x87\\xBA\\xF0\\x9F...'

python program ,insert data in mysql table:
is_exist_table_sql = "SHOW TABLES LIKE 'fb_public_figure_posts'"
if cur.execute(is_exist_table_sql) == 0:
create_sql = """CREATE TABLE fb_public_figure_posts(Post_ID varchar(32), Permalink varchar(128), Create_time varchar(32), Updated_time varchar(32), Author varchar(32),
Author_ID bigint, Message text, Link varchar(1024), Likes int, Comments int, pf_ID bigint, foreign key(pf_ID) references fb_public_figure_info(ID))"""
cur.execute(create_sql)
db_data = posts
if type == "public_figure_posts":
for item in db_data:
if "'" in item["message"]:
item["message"] = str(item["message"]).replace("'","\\\'")
elif '"' in item["message"]:
item["message"] = str(item["message"]).replace('"','\\\"')
is_exist_id_sql = "select * from fb_public_figure_posts where Post_ID = '" + item['id'] + "'"
if cur.execute(is_exist_id_sql) == 0:
insert_sql = "INSERT INTO fb_public_figure_posts VALUES ('{0}','{1}','{2}','{3}','{4}',{5},'{6}','{7}',{8},{9},{10})".format(item['id'],item['permalink_url'],item['created_time'],item['updated_time'],item['from']['name'],item['from']['id'],item['message'],item['link'],item['likes']['summary']['total_count'],item['comments']['summary']['total_count'],public_figure_id)
print(insert_sql)
cur.execute(insert_sql)
when running , it errors:
pymysql.err.InternalError: (1366, "Incorrect string value: '\\xF0\\x9F\\x87\\xBA\\xF0\\x9F...' for column 'Message' at row 1")
it point that the sentence error:
Traceback (most recent call last):
File "C:/Python/PyCharmProject/FaceBookCrawl/FBCrawl.py", line 286, in <module>
publicfigure_download.public_figure_posts_storage(public_figure_name)
INSERT INTO fb_public_figure_posts VALUES ('153080620724_10158392447835725','https://www.facebook.com/DonaldTrump/posts/10158392447835725:0','2017-01-01T04:59:07+0000','2017-01-23T19:52:49+0000','Donald J. Trump',153080620724,'‪TO ALL AMERICANS-‬
File "C:\Python\PyCharmProject\FaceBookCrawl\publicfigure_download.py", line 103, in public_figure_posts_storage
‪#HappyNewYear & many blessings to you all! Looking forward to a wonderful & prosperous 2017 as we work together to #MAGA🇺🇸‬','https://www.facebook.com/DonaldTrump/photos/a.488852220724.393301.153080620724/10158392447835725/?type=3',158710,11045,153080620724)
mysql_manage().public_figure_db_manage(type, posts, public_figure_id, public_figure_name)
File "C:\Python\PyCharmProject\FaceBookCrawl\database_manage.py", line 47, in public_figure_db_manage
cur.execute(insert_sql)
File "C:\Python\Python36\lib\site-packages\pymysql\cursors.py", line 166, in execute
result = self._query(query)
File "C:\Python\Python36\lib\site-packages\pymysql\cursors.py", line 322, in _query
conn.query(q)
File "C:\Python\Python36\lib\site-packages\pymysql\connections.py", line 835, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "C:\Python\Python36\lib\site-packages\pymysql\connections.py", line 1019, in _read_query_result
result.read()
File "C:\Python\Python36\lib\site-packages\pymysql\connections.py", line 1302, in read
first_packet = self.connection._read_packet()
File "C:\Python\Python36\lib\site-packages\pymysql\connections.py", line 981, in _read_packet
packet.check_error()
File "C:\Python\Python36\lib\site-packages\pymysql\connections.py", line 393, in check_error
err.raise_mysql_exception(self._data)
File "C:\Python\Python36\lib\site-packages\pymysql\err.py", line 107, in raise_mysql_exception
raise errorclass(errno, errval)
pymysql.err.InternalError: (1366, "Incorrect string value: '\\xF0\\x9F\\x87\\xBA\\xF0\\x9F...' for column 'Message' at row 1")
Process finished with exit code 1
in conclusion, it shows:
pymysql.err.InternalError: (1366, "Incorrect string value: '\\xF0\\x9F\\x87\\xBA\\xF0\\x9F...' for column 'Message' at row 1")
could you please help me for that

You actually have two problems here. The first one is that you're trying to insert a string containing characters that are not supported by your database/table/field encoding - the catch is that MySQL's "utf-8" encoding is NOT really utf8 compliant (surprise, surprise...). The linked SO question gives full explanations, you can also check this for a migration procedure.
Your second issue - which AFAICT isn't responsible for the problem you describe but will cause quite a few other problems anyway - is that you're not using your db connector properly. This makes your code uselessly complicated and quite brittle - proper escaping is a pain as you might have noticed, so better to let the db connector do the job -, but also opens your app to SQL injection attacks. The cure here is simple: use prepared statements. This is actually much simpler than not using them and will kill two birds with one stone.

pymssql bulk insert error (python)

Trying to bulk insert a CSV file using pymssql here's the code:
conn = pymssql.connect(host='server', user='user', password='secret', database='My_Dev')
cur = conn.cursor()
load = 'BULK INSERT TempStaging FROM \'/home/dross/python/scripts/var/csv/' + f + '.csv\' WITH (FIRSTROW = 1,FIELDTERMINATOR = ',',ROWTERMINATOR = \'\\n\') GO")'
cur.execute(load)
When executing get following error:
Traceback (most recent call last):
File "./uploadResults.py", line 46, in <module>
cur.execute(sweepload)
File "pymssql.pyx", line 447, in pymssql.Cursor.execute (pymssql.c:7092)
File "_mssql.pyx", line 1009, in _mssql.MSSQLConnection.execute_query (_mssql.c:11585)
File "_mssql.pyx", line 1040, in _mssql.MSSQLConnection.execute_query (_mssql.c:11459)
File "_mssql.pyx", line 1160, in _mssql.MSSQLConnection.format_and_run_query (_mssql.c:12652)
File "_mssql.pyx", line 203, in _mssql.ensure_bytes (_mssql.c:2733)
AttributeError: 'tuple' object has no attribute 'encode'
Line 46 is cur.execute line

Note that .format() could allow sql injection, but if you control the filename then it's not that bad (not sure if a parameter would work here).
Also, you should use triple-quoted strings when dealing with SQL, your life will be so much better. Like this:
load = '''BULK INSERT TempStaging FROM /home/dross/python/scripts/var/csv/{}.csv WITH (FIRSTROW=1, FIELDTERMINATOR=',', ROWTERMINATOR='\n')'''.format(filename)
Being triple quoted, you can also break it up to make it easier to read:
load = '''
BULK INSERT TempStaging
FROM /home/dross/python/scripts/var/csv/{}.csv
WITH (
FIRSTROW=1
, FIELDTERMINATOR=','
, ROWTERMINATOR='\n'
)
'''.format(filename)

You should defined the string as below:
load = "BULK INSERT TempStaging FROM /home/dross/python/scripts/var/csv/" + f + ".csv WITH ( FIRSTROW=1 , FIELDTERMINATOR=',' , ROWTERMINATOR='\\n')"

Inserting language characters in Django

I am following the link http://code.google.com/p/django-multilingual-model/ Basically i am trying to insert hindi characters into mysql db
I have created the files in the test directory as in the above said link and using django version 1.1 an dpython version is 2.4.3
I geta error message as the following.
from testlanguages import Language,BookTranslation,Book
>>> lang_pl = Language(code="pl", name="Polish")
>>> lang_pl.save()
>>> book_pl.language = lang_pl
>>> book_pl.save()
>>> lang_hi = Language(code="hi", name="Hindi")
>>> lang_hi.save()
>>> book_hi = BookTranslation()
>>> book_hi.title = "का सफल प्रक्षेपण किया है. इस राकेट ने पाँच" Sum characters as in bbc.co.uk/hindi
>>> book_hi.description = "का सफल प्रक्षेपण किया है. इस राकेट ने पाँच" Sum characters as in bbc.co.uk/hindi
>>> book_hi.model = book
>>> book_hi.language = lang_hi
>>> book_hi.save()
Traceback (most recent call last):
File "<console>", line 1, in ?
File "/opt/project/django/django/db/models/base.py", line 410, in save
self.save_base(force_insert=force_insert, force_update=force_update)
File "/opt/project/django/django/db/models/base.py", line 495, in save_base
result = manager._insert(values, return_id=update_pk)
File "/opt/project/django/django/db/models/manager.py", line 177, in _insert
return insert_query(self.model, values, **kwargs)
File "/opt/project/django/django/db/models/query.py", line 1087, in insert_query
return query.execute_sql(return_id)
File "/opt/project/django/django/db/models/sql/subqueries.py", line 320, in execute_sql
cursor = super(InsertQuery, self).execute_sql(None)
File "/opt/project/django/django/db/models/sql/query.py", line 2369, in execute_sql
cursor.execute(sql, params)
File "/opt/project/django/django/db/backends/util.py", line 19, in execute
return self.cursor.execute(sql, params)
File "/opt/project/django/django/db/backends/mysql/base.py", line 84, in execute
return self.cursor.execute(query, args)
File "/usr/lib/python2.4/site-packages/MySQLdb/cursors.py", line 165, in execute
self._warning_check()
File "/usr/lib/python2.4/site-packages/MySQLdb/cursors.py", line 80, in _warning_check
warn(w[-1], self.Warning, 3)
File "/usr/lib/python2.4/warnings.py", line 61, in warn
warn_explicit(message, category, filename, lineno, module, registry)
File "/usr/lib/python2.4/warnings.py", line 96, in warn_explicit
raise message
Warning: Incorrect string value: '\xE0\xA4\x95\xE0\xA4\xBE...' for column 'title' at row 1
How to resolve this.
Is there any easy method to insert the special characters into DB

Stop using bytestrings. Start using unicode.
book_hi.title = u"का सफल प्रक्षेपण किया है. इस राकेट ने पाँच"

conn = MySQLdb.connect( host = "localhost",
user = "root",
passwd = "",
db = "test");
conn.set_character_set('utf8')
this code will solve you problem
Note *. conn.set_character_set('utf8') is notable part

This is most likely a MySQL problem, which doesn't always support Unicode characters by default.
You should probably declare the DEFAULT CHARSET to utf8 and DEFAULT COLLATION to utf8_general_ci on your database (use the ALTER DATABASE or modify the CREATE DATABASE statements, as explained here).