I have the following code:
#!/usr/bin/python
# coding=UTF-8
import cx_Oracle
def oracle_connection(user, passwd, host, port, service):
oracle_con_details = user+'/'+passwd+'#'+host+':'+port+'/'+service
try:
oracle_connection = cx_Oracle.connect(oracle_con_details)
except cx_Oracle.DatabaseError as e:
error, = e.args
if error.code == 1017:
log.warning('Please check your credentials.')
else:
log.error('Database connection error: ')
log.error(e)
return oracle_connection
user_oracle = "user"
passw_oracle = "pass"
host_oracle = "host"
port_oracle = "port"
service_oracle = "service"
con_oracle = oracle_connection(user_oracle, passw_oracle, host_oracle, port_oracle, service_oracle)
query = """ SELECT COUNT(*) FROM TABLE WHERE MYDATA = 'REUNIÓN'"""
cursor_oracle = con_oracle.cursor()
cursor_oracle.execute(query)
data_tuple = cursor_oracle.fetchall()
Of course, Oracle credentials and query are just examples. Notice the query has 'Ó' character. This is the one giving me the following error:
UnicodeEncodeError: 'ascii' codec can't encode character u'\xd3' in
position 49: ordinal not in range(128)
I've tried the solutions from some other questions here:
query.decode('utf-8')
query.encode('utf-8')
query.decode('unicode')
query.encode('unicode')
I understand my string (query) is encoded in unicode but I just don't understand why decoding it in utf-8 doesn't work.
Because of this my query doesn't get to Oracle how it should.
ADDITIONAL INFO:
Based on this answer I thought mystring.encode('utf-8) would work.
I cheked the type of my string with this method just in case and the result is 'ordinary string'.
Adding this to my python code solved it.
import os
os.environ["NLS_LANG"] = "SPANISH_SPAIN.UTF8"
Related
I am a beginner in Python. I am trying to insert the following data in sqlite db using Python 3.4.
('config.xml', '09/12/2017 10:33:55 PM', 466, 'C:Users\ron\Downloads\folder');
But I am getting an error
SyntaxError: (unicode error) 'unicodeescape' codec can't decode bytes in position 23-26: truncated \UXXXXXXXX escape
I think this is because of this character - \
How can I tell sqlite to escape this character.
Code --
def create_filedetail(conn, filedetail):
"""
Create a new file detail into the filedetail table
:param conn:
:param filedetail:
:return: filedetail id
"""
sql = ''' INSERT INTO filedetail(filename,modified_date,filesize,filepath)
VALUES(?,?,?,?) '''
cur = conn.cursor()
cur.execute(sql, filedetail)
return cur.lastrowid
def main():
database = r"C:\Users\ron\Documents\sqlitedb\filedb.db"
sql_create_file_detail_table = """ CREATE TABLE IF NOT EXISTS filedetail (
id integer PRIMARY KEY,
filename text NOT NULL,
modified_date text,
filesize integer,
filepath text
); """
conn = create_connection(database)
if conn is not None:
# create filedetail table
create_table(conn, sql_create_file_detail_table)
with conn:
# create a new filedetail
filedetail = ('config.xml', '09/12/2017 10:33:55 PM', 466, "C:Users\ron\Downloads\folder");
filedetail_id = create_filedetail(conn, filedetail)
else:
print("Error! cannot create the database connection.")
Any help is highly appreciated. Thanks in advance.
You can see that there is a similar issue in this post:
What exactly do "u" and "r" string flags do, and what are raw string literals?
Basically, you can create a string literal by prepending an r to your string.
Look at this example. It returns an invalid character:
>>> mypath = "C:Users\ron\Downloads\folder"
>>> mypath
'C:Users\ron\\Downloads\x0colder'
However, if you use the string literals:
>>> mypath = r"C:Users\ron\Downloads\folder"
>>> mypath
'C:Users\\ron\\Downloads\\folder'
You can then insert this new string into your SQL table.
I use python to connect hive & retrieve the data into pandas, but its giving an error:
pyhive.exc.OperationalError: TExecuteStatementResp
my code:
# -*- coding: utf-8 -*-
from pyhive import hive
from impala.util import as_pandas
from string import Template
config = {
'host': '127.0.0.1',
'database': 'default'
}
def get_conn(conf):
conn = hive.connect(**conf)
return conn
def execute_hql(hql, params = None):
conn = get_conn(config)
cursor = conn.cursor()
hql = Template(hql).substitute(params)
cursor.execute(hql)
df = as_pandas(cursor)
return df
test.py
# -*- coding: utf-8 -*-
from pyhive import hive
from impala.util import as_pandas
import DB.hive_engines
hql = """
SELECT
keywords,
count(keywords)
FROM
table
WHERE
eventname = 'xxx' AND
cdate >= '$start_date' AND
cdate <= '$end_date'
GROUP BY
keywords
"""
if __name__ == '__main__':
params = {'start_date': '2016-04-01', 'end_date': '2016-04-03'}
df = DB.hive_engines.execute_hql(hql, params)
print df
exception message:
pyhive.exc.OperationalError: TExecuteStatementResp(status=TStatus(errorCode=1, errorMessage='Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask', sqlState='08S01', infoMessages=['*org.apache.hive.service.cli.HiveSQLException:Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask:28:27', 'org.apache.hive.service.cli.operation.Operation:toSQLException:Operation.java:326', 'org.apache.hive.service.cli.operation.SQLOperation:runQuery:SQLOperation.java:146', 'org.apache.hive.service.cli.operation.SQLOperation:runInternal:SQLOperation.java:173', 'org.apache.hive.service.cli.operation.Operation:run:Operation.java:268', 'org.apache.hive.service.cli.session.HiveSessionImpl:executeStatementInternal:HiveSessionImpl.java:410', 'org.apache.hive.service.cli.session.HiveSessionImpl:executeStatement:HiveSessionImpl.java:391', 'sun.reflect.GeneratedMethodAccessor31:invoke::-1', 'sun.reflect.DelegatingMethodAccessorImpl:invoke:DelegatingMethodAccessorImpl.java:43', 'java.lang.reflect.Method:invoke:Method.java:606', 'org.apache.hive.service.cli.session.HiveSessionProxy:invoke:HiveSessionProxy.java:78', 'org.apache.hive.service.cli.session.HiveSessionProxy:access$000:HiveSessionProxy.java:36', 'org.apache.hive.service.cli.session.HiveSessionProxy$1:run:HiveSessionProxy.java:63', 'java.security.AccessController:doPrivileged:AccessController.java:-2', 'javax.security.auth.Subject:doAs:Subject.java:415', 'org.apache.hadoop.security.UserGroupInformation:doAs:UserGroupInformation.java:1671', 'org.apache.hive.service.cli.session.HiveSessionProxy:invoke:HiveSessionProxy.java:59', 'com.sun.proxy.$Proxy27:executeStatement::-1', 'org.apache.hive.service.cli.CLIService:executeStatement:CLIService.java:245', 'org.apache.hive.service.cli.thrift.ThriftCLIService:ExecuteStatement:ThriftCLIService.java:509', 'org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement:getResult:TCLIService.java:1313', 'org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement:getResult:TCLIService.java:1298', 'org.apache.thrift.ProcessFunction:process:ProcessFunction.java:39', 'org.apache.thrift.TBaseProcessor:process:TBaseProcessor.java:39', 'org.apache.hive.service.auth.TSetIpAddressProcessor:process:TSetIpAddressProcessor.java:56', 'org.apache.thrift.server.TThreadPoolServer$WorkerProcess:run:TThreadPoolServer.java:285', 'java.util.concurrent.ThreadPoolExecutor:runWorker:ThreadPoolExecutor.java:1145', 'java.util.concurrent.ThreadPoolExecutor$Worker:run:ThreadPoolExecutor.java:615', 'java.lang.Thread:run:Thread.java:745'], statusCode=3), operationHandle=None)
Thanks!
Following this discussion, I used a valid username while creating the connection and that solved the problem.
For the sake of completeness of this answer, I am copy pasting the suggested code from the above mentioned forum. Please note the valid username there.
from pyhive import hive
conn = hive.Connection(host='<myhost>',
port='<myport>',
database='spin1',
username='<a valid user>') # IMPORTANT**
cursor = conn.cursor()
print cursor.fetchall()
In absence of the valid username, I was hitting the same exception mentioned in the question.
I search in an MS Active Directory with Python and I stumbled upon an issue with the encoding of an answer I get.
I use the code below to recursively go through the AD
import ldap
import pickle
class SearchAD():
def __init__(self):
self.l = ldap.initialize("ldap://ldap.example.com")
self.l.protocol_version = ldap.VERSION3
self.l.set_option(ldap.OPT_REFERRALS, 0)
bind = self.l.simple_bind_s("user", "password")
self.base = "DC=example,DC=com"
self.all = list()
def searchmgr(self, criteria, m):
print criteria
m += 1
attributes = ['dn', 'title']
result = self.l.search_s(self.base, ldap.SCOPE_SUBTREE, u'manager='+criteria, attributes)
for u in result:
cn, t = u
if cn is not None and "Disabled Users" not in cn and t.get('title'):
self.all.append({'dn': cn, 'title': t['title'][0], 'm': m})
self.searchmgr(cn, m)
s = SearchAD()
s.searchmgr("CN=EXAMPLE Top,DC=example,DC=com", 0)
with open("dir.pickle", "wb") as f:
pickle.dump(s.all, f)
and get
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position
16: ordinal not in range(128)
when calling
result = self.l.search_s(self.base, ldap.SCOPE_SUBTREE, 'manager='+criteria, attributes)
The search works fine (I get the expected output via the print) until the string 'CN=DOE Marie-h\xc3\xa9l\xc3\xa8ne,OU=User Accounts,...' which translates to 'CN=DOE Marie-hélène,OU=User Accounts,...'.
This is expected as AD returns its results in UTF-8
I therefore tried to .encode('utf-8') the string and use
result = self.l.search_s(self.base, ldap.SCOPE_SUBTREE, u'manager='+criteria.encode('utf-8'), attributes)
but I get the same error.
How should I handle the result returned by AD?
I am trying to work with sqlite on python:
from pysqlite2 import dbapi2 as sqlite
con = sqlite.connect('/home/argon/super.db')
cur = con.cursor()
cur.execute('select * from notes')
for i in cur.fetchall():
print i[2]
And I sometimes get something like this (I am from Russia):
ÐÑÐ²ÐµÑ etc...
And if I pass this string to this function(it helped me in other projects):
def unescape(text):
def fixup(m):
text = m.group(0)
if text[:2] == "&#":
# character reference
try:
if text[:3] == "&#x":
return unichr(int(text[3:-1], 16))
else:
return unichr(int(text[2:-1]))
except ValueError:
pass
else:
# named entity
try:
text = unichr(htmlentitydefs.name2codepoint[text[1:-1]])
except KeyError:
pass
return text # leave as is
return re.sub("&#?\w+;", fixup, text)
I get even more weird result:
ÐÑвеÑиÑÑ Ñ ÑиÑиÑованием etc
What should I do to get normal Cyrillic symbols?
Ð looks like a UTF-8 byte pair for \xD0\x9E, or \u1054. Better known as the cyrillic character О (Capital O).
In other words, you have strangely encoded UTF-8 data on your hand. Turn the { digits into bytes (chr(208) would do) then decode from UTF-8:
>>> (chr(208) + chr(158)).decode('utf-8')
u'\u1054'
>>> print (chr(208) + chr(158)).decode('utf-8')
О
>>> print (chr(208) + chr(158) + chr(209) + chr(130) + chr(208) + chr(178)).decode('utf-8')
Отв
i have the following:
ora_wet = oracle_connection()
cursor = ora_wet.cursor()
sqlQuery = u"SELECT * FROM web_cities WHERE cty_name = 'София'"
cursor.execute(sqlQuery)
sqlResult = cursor.fetchone()
When I do this I get the following error:
TypeError: expecting None or a string on line 18 which is the cursor.execute(sqlQuery)
If I make the query non-unicode (without the u) it goes through but it returns nothing
edit: in reply to first comment:
NLS_LANGUAGE is BULGARIAN,
NLS_CHARACTERSET is CL8MSWIN1251
language is Python...
yes there is a record with cty_name = 'София'
connection is just:
def oracle_connection():
return cx_Oracle.connect('user/pass#server')
ora_wet = oracle_connection()