Original Code:
import sys
import os
import latexmake
import mysql.connector
conn = mysql.connector.connect(user='root',password='oilwell',host='localhost',database='sqlpush1')
with conn:
mycursor = conn.cursor()
mycursor=execute("SELECT DATE,oil,gas,oilprice,gasprice,totrev FROM results WHERE DATE BETWEEN '2011-01-01' AND '2027-12-01'")
rows = mycursor.fetchall()
a.write("\\documentclass{standalone}\\usepackage{booktabs}\n\n\\usepackage{siunitx}\r \n\
\r\n\\begin{document}\r\n\\begin{tabular}{ccS[table-format = 5.2]} \\\\ \\toprule\r")
a.write("Date & Oil & Gas & Oil price & Gas price & Total Revenue \\\\ \\midrule \r")
for row in rows:
a = open("testtest.tex", "w")
a.write("" + str(row[0]) + " & " + str(row[1]) + " & " + str(row[2]) + " & " + str(row[3]) + " & " + str(row[4]) + " & " + str(row[5]) + " \\\\ \r")
a.write("\\bottomrule \\end{tabular}\r\\end{document}")
a.close
print (os.path.getsize("testtest.tex"))
os.system('latexmk.py -q testtest.tex')
mycursor.close()
conn.close()
a.close()
After run by IDLE, and red error pop up like
Traceback (most recent call last):
File "C:\Users\Cheng XXXX\Desktop\tabletest.py", line 8, in <module>
with conn:
AttributeError: __exit__
I checked the file and cannot file mistake, need help.
You are trying to use the connection as a context manager:
with conn:
This object doesn't implement the necessary methods to be used like that; it is not a context manager, as it is missing (at least) the __exit__ method.
If you are reading a tutorial or documentation that uses a different MySQL library, be aware that this feature may be supported by some libraries, just not this one. The MySQLdb project does support it, for example.
For your specific case, you don't even need to use the with conn: line at all; you are not making any changes to the database, no commit is required anywhere. You can safely remove the with conn: line (unindent everything under it one step). Otherwise you can replace the context manager with a manual conn.commit() elsewhere.
Alternatively, you can create your own context manager for this use-case, using the #contextlib.contextmanager() decorator:
from contextlib import contextmanager
#contextmanager
def manage_transaction(conn, *args, **kw):
exc = False
try:
try:
conn.start_transaction(*args, **kw)
yield conn.cursor()
except:
exc = True
conn.rollback()
finally:
if not exc:
conn.commit()
and use this as:
with manage_transaction(conn) as cursor:
# do things, including creating extra cursors
where you can pass in extra arguments for the connection.start_transaction() call.
Related
Have a problem with my ETL process.
I've got ETL process, written in python and it works great, but operations
starts one after another, so the whole process lasts much time.
I'm slightly new in Apache Airflow, but I've made a DUG and there is a problem
with him)
I get a mistake:
File "/usr/lib/python3.8/encodings/utf_16_le.py", line 15, in decode
def decode(input, errors='strict'):
File "/usr/local/lib/python3.8/dist-packages/airflow/models/taskinstance.py", line 1543, in signal_handler
raise AirflowException("Task received SIGTERM signal")
airflow.exceptions.AirflowException: Task received SIGTERM signal
The above exception was the direct cause of the following exception:
airflow.exceptions.AirflowException: decoding with 'utf-16le' codec failed (AirflowException: Task received SIGTERM signal)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py", line 1705, in _execute_context
self.dialect.do_execute(
File "/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/default.py", line 716, in do_execute
cursor.execute(statement, parameters)
SystemError: <class 'pyodbc.Error'> returned a result with an error set
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py", line 896, in _rollback_impl
self.engine.dialect.do_rollback(self.connection)
File "/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/default.py", line 666, in do_rollback
dbapi_connection.rollback()
pyodbc.OperationalError: ('08S01', '[08S01] [Microsoft][ODBC Driver 18 for SQL Server]Communication link failure (0) (SQLEndTran)')
Here is a code of my Task. There can be up to 10 connections at once:
def update_from_gladiator_ost(market_id):
query = "DELETE from [stage].[dbo].[rests_by_docs_temp] where market_id = %d" % market_id
execute_query_dwh(query)
engine = dwh_conn()
connection = engine.raw_connection()
abc = connection.cursor()
# abc.execute("DELETE from [stage].[dbo].[sell_movement_temp]; DELETE from [stage].[dbo].[rests_by_docs_temp]")
df_op = pd.read_sql(
"SET NOCOUNT ON exec [dbo].[mp_report_finance_agent_enhanced_basis_transport_royalty_NC_ost_by_docs4] #pmarket_id = %d, #pstart_date = '%s', #pend_date = '%s', #pselect = '1'" % (
market_id, z, w), gladiator_conn())
df_op = df_op.fillna(value=0)
for row_count in range(0, df_op.shape[0]):
chunk = df_op.iloc[row_count:row_count + 1, :].values.tolist()
tuple_of_tuples = tuple(tuple(x) for x in chunk)
abc.executemany(
"insert into stage.dbo.rests_by_docs_temp" + " ([date_start],[market_id],[good_id],[agent_id],[doc_id],[tstart_qty],[tstart_amt],[IMP],[doc_name]) values (?,?,?,?,?,?,?,?,?)",
tuple_of_tuples)
abc.commit()
connection.close()
As you see, I get data from database and INSERT it in my DWH
And here is my connections:
def dwh_conn():
mySQL = '192.168.240.1'
myDB = 'DWH'
login = 'sa'
PWD = '....'
Encrypt = 'No'
Certificate = 'Yes'
params = urllib.parse.quote_plus("DRIVER={ODBC Driver 18 for SQL Server};"
"SERVER=" + mySQL + ";"
"SERVER=" + mySQL + ";"
"Port=1433" + ";"
"DATABASE=" + myDB + ";"
"UID=" + login + ";"
"PWD=" + PWD + ";"
"Encrypt=" + Encrypt + ";"
"TrustServerCertificate=" + Certificate + ";")
engine = sa.create_engine('mssql+pyodbc:///?odbc_connect={}?charset=utf8mb4'.format(params), fast_executemany=True)
return engine
def gladiator_conn():
mySQL = '...'
myDB = '...'
login = '...'
PWD = '...'
Encrypt = 'No'
Certificate = 'Yes'
params = urllib.parse.quote_plus("DRIVER={ODBC Driver 18 for SQL Server};"
"SERVER=" + mySQL + ";"
"Port=1433" + ";"
"DATABASE=" + myDB + ";"
"UID=" + login + ";"
"PWD=" + PWD + ";"
"Encrypt=" + Encrypt + ";"
"TrustServerCertificate=" + Certificate + ";")
engine = sa.create_engine('mssql+pyodbc:///?odbc_connect={}?charset=utf8mb4'.format(params), fast_executemany=True)
return engine
I think the problem is in unixODBC. Because when I do the whole code in Pycharm on Windows - everythong is fine.
But on docker Ubuntu/Airflow - it sometimes fails.
I can restart the task which failed and it can go fine but can fail again
updated:
I guess, I Found one solution but I cant realize it on my case.
def decode_sketchy_utf16(raw_bytes):
s = raw_bytes.decode("utf-16le", "ignore")
try:
n = s.index('\u0000')
s = s[:n] # respect null terminator
except ValueError:
pass
return s
# ...
prev_converter = cnxn.get_output_converter(pyodbc.SQL_WVARCHAR)
cnxn.add_output_converter(pyodbc.SQL_WVARCHAR, decode_sketchy_utf16)
col_info = crsr.columns("Clients").fetchall()
cnxn.add_output_converter(pyodbc.SQL_WVARCHAR, prev_converter) # restore previous behaviour
Help me how to make it work in my code? Where should I implement it?
Found an answer. These problem rises when I'm lack of memory (operative). Especially when several containers on, it might go to this error
I have this piece of code that collects data from a HAT connected to a Raspberry.
When run it gives this error:
[51.57, 22.30, 1002.01]
Traceback (most recent call last):
File "dbWriter.py", line 45, in <module>
write2DB(record)
File "dbWriter.py", line 26, in write2DB
assert len(values) == 3
AssertionError
I am by no means a programmer, i just fiddle around. It is meant to save 'record' to a database, which is then read and updated in realtime on an apache2 server. All help appreciated.
import mysql.connector
from itertools import repeat
import sys
import bme680
import time
try:
sensor = bme680.BME680(bme680.I2C_ADDR_PRIMARY)
except IOError:
sensor = bme680.BME680(bme680.I2C_ADDR_SECONDARY)
sensor.set_humidity_oversample(bme680.OS_2X)
sensor.set_pressure_oversample(bme680.OS_4X)
sensor.set_temperature_oversample(bme680.OS_8X)
sensor.set_filter(bme680.FILTER_SIZE_3)
mydb = mysql.connector.connect(
host='localhost',
user='pi',
passwd='pass',
database='weatherDB'
)
mycursor = mydb.cursor()
def write2DB(values):
assert len(values) == 3
sqlText = '''INSERT INTO data(humidity, temperature, pressure) VALUES({},{}, })'''.format(values[0], values[1], values[2])
mycursor.execute(sqlText)
mydb.commit()
for _ in repeat(None):
sensor.get_sensor_data()
output_humi = '{0:.2f}'.format(
sensor.data.humidity)
output_temp = '{0:.2f}'.format(
sensor.data.temperature)
output_pres = '{0:.2f}'.format(
sensor.data.pressure)
record = []
record = ('[' + (output_humi) + ', ' + (output_temp) + ', ' + (output_pres) + ']')
print(record)
write2DB(record)
time.sleep(10)
pass
You have:
record = ('[' + (output_humi) + ', ' + (output_temp) + ', ' + (output_pres) + ']')
record evaluates to a single string, not a list of 3 elements and hence your exception.
Change the above to:
record = [output_humi, output_temp, output_pres]
You are also missing a { in your format specification. It should be:
sqlText = '''INSERT INTO data(humidity, temperature, pressure) VALUES({},{}, {})'''.format(values[0], values[1], values[2])
An alternative would be to use a prepared statement:
sqlText = 'INSERT INTO data(humidity, temperature, pressure) VALUES(%s, %s, %s)'
mycursor.execute(sqlText, values)
In the above case you will be passing actual strings as the values. I don't know how the columns are defined, but no matter. If they are defined as floating point or decimal values, the strings will be converted to the correct type.
I'm trying to run a python function on the cursor.execute parameter but it just throws me this error.
I'm using psycopg2
Traceback (most recent call last):
File "cliente.py", line 55, in <module>
cursorDB.execute(get_datos_animal('falsa'))
psycopg2.errors.UndefinedColumn: column "falsa" does not exist
LINE 1: ...e, clasificacion FROM animales WHERE animales.hierro = falsa
and my python function is this one
def get_datos_animal(hierro_v):
return "SELECT hierro, registro, nombre, fecha_nacimiento, raza, sexo, hierro_madre, hierro_padre, clasificacion FROM animales WHERE animales.hierro = " + str(hierro_v)
any idea what i´m doing wrong?
Have several functions like this with same errors.
Use the automatic parameter quoting provided by your connection to ensure that values in queries are always quoted correctly, and to avoid SQL injection attacks.
stmt = """SELECT hierro, registro, nombre, fecha_nacimiento, raza, sexo, hierro_madre, hierro_padre, clasificacion
FROM animales
WHERE animales.hierro = %s"""
cursor.execute(stmt, (hierro_v,))
In postgres if you pass value without quotes it will treat that as column name.
Try this:
def get_datos_animal(hierro_v):
return "SELECT hierro, registro, nombre, fecha_nacimiento, raza, sexo, hierro_madre, hierro_padre, clasificacion FROM animales WHERE animales.hierro = '"+str(hierro_v)+"'"
EDIT to format:
This is the original code
from __future__ import print_function
import socket
import sys
def socket_accept():
conn, address = s.accept()
print("Connection has been established | " + "IP " + address[0] + "| Port " + str(address[1]))
send_commands(conn)
conn.close()
def send_commands(conn):
while True:
cmd = raw_input()
if cmd == 'quit':
conn.close()
s.close()
sys.exit()
if len(str.encode(cmd)) > 0:
conn.send(str.encode(cmd))
client_response = str(conn.recv(1024), "utf-8")
print(client_response, end ="")
def main():
socket_accept()
main()
I am getting this error “TypeError: str() takes at most 1 argument (2 given)” at “client_response” variable
You have your error here:
client_response = str(conn.recv(1024), "utf-8")
Just change it to:
client_response = str(conn.recv(1024)).encode("utf-8")
On the second to last line you're passing two arguments to the str function, although the str function only takes a single argument in Python 2. It does in fact take up to three arguments in python 3
https://docs.python.org/2.7/library/functions.html?highlight=str#str
https://docs.python.org/3.6/library/functions.html?highlight=str#str
So you're either trying to inadvertaetly run python 3 code in a python 2 interpreter or you're looking at the wrong language documentation.
So either use #franciscosolimas's answer, if you're using python 2, or make sure you're using python 3, if the latter you might also want to add a keyword argument just to make sure you know what's happening in the future
client_response = str(conn.recv(1024), encoding="utf-8")
3 arguments, 5 given
I got a similar error, may not be the same here (as the op) but, it was simple enough fix and wanted to share, since I ended up here from my searches on the error.
Traceback (most recent call last):
File "queries.py", line 50, in <module>
"WHERE ao.type='u';")
TypeError: str() takes at most 3 arguments (5 given)`
What fixed it for me in python3 was converting my ,'s to +
Error:
str("SELECT s.name + '.' + ao.name, s.name"
"FROM sys.all_objects ao",
"INNER JOIN sys.schemas s",
"ON s.schema_id = ao.schema_id",
"WHERE ao.type='u';"))
Fixed:
str("SELECT s.name + '.' + ao.name, s.name " +
"FROM sys.all_objects ao " +
"INNER JOIN sys.schemas s " +
"ON s.schema_id = ao.schema_id " +
"WHERE ao.type='u';")
I had to add my own spaces so the passed query would work.
As the commas were doing that in python...
Thoughts & my educated guess:
looks like in my case it got caught up trying to evaluate in bash/python a litteral u'
To my knowledge this break could be in bash because there is no command called u and/or in python u' is you trying to unicode an incomplete string. Either way it broke and wanted to share my fix.
Cheers!
~JayRizzo
I've learn some basics about python-mysqldb ,when I want to define anther function for query,I have to write (connect ,cursor...try ..) repeatedly
so I want to design a template like jdbcTemplate (Java EE, Spring)
my code is:
def DBV():
def templateFN(fn):
logging.basicConfig(level=logging.INFO)
log = logging.getLogger('DB')
conn = MySQLdb.connect(user='root',passwd='247326',db='lucky',charset="utf8",cursorclass=MySQLdb.cursors.DictCursor);
cursor = conn.cursor()
def wrap(data=None):
try:
return fn(cursor=cursor,data=data)
#conn.commit()
except Exception ,e:
conn.rollback()
log.error('%s, transaction rollback',e)
finally:
cursor.close()
conn.close()
return wrap
class DB():
#templateFN
def insertTest(self,cursor,data=None):
data = {
'field':'this is a test',
'name':'this is a name'
}
return cursor.execute('insert into test(field,name) values(%(field)s,%(name)s)',data)
return DB()
db = DBV()
print 'return value',db.insertTest(data="ok")
Traceback (most recent call last):
File "D:\WorkSpaces\Aptana Studio 3 Workspace\VLuck\src\com\test.py", line 164, in
print 'return value',db.insertTest(data="ok")
TypeError: wrap() got multiple values for keyword argument 'data'
but failed,how should I do it right
Here's a solution I came up with, inspired by another answer:
def connect_mysql(func):
# Assign value of 'self' to be default func
func.func_defaults = func.func_defaults[:-1] + (func,)
func._cnx = mysql.connector.connect(**CONFIG)
func._cursor = func._cnx.cursor()
return func
#connect_mysql
def test(data, self=None):
self._cursor.execute("SELECT %(c1)s", data)
print(self._cursor.fetchall())
test({'c1': 1})