I have mysql data like this
id | name | time_stamp |
which time_stamp field contains data in datetime format
So this is the code to export that mysql data to xls :
w = Workbook()
ws = w.add_sheet("Sheet Title") # worksheet name
# Database connection
con = MySQLdb.connect(host='localhost', user='', passwd='', db='dbtest')
cur = con.cursor()
cur.execute("SELECT name, time_stamp FROM log")
# iterate row and col
rowNum = 0
colNum = 0
# Print all to worksheet
for row in cur.fetchall():
ws.write(rowNum, colNum, row)
rowNum =+1
colNum =+1
and this is the results :
Traceback (most recent call last)
File "/usr/share/pyshared/flask/app.py", line 1836, in __call__
return self.wsgi_app(environ, start_response)
File "/usr/share/pyshared/flask/app.py", line 1820, in wsgi_app
response = self.make_response(self.handle_exception(e))
File "/usr/share/pyshared/flask/app.py", line 1403, in handle_exception
reraise(exc_type, exc_value, tb)
File "/usr/share/pyshared/flask/app.py", line 1817, in wsgi_app
response = self.full_dispatch_request()
File "/usr/share/pyshared/flask/app.py", line 1477, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/share/pyshared/flask/app.py", line 1381, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/usr/share/pyshared/flask/app.py", line 1475, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/share/pyshared/flask/app.py", line 1461, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/home/pi/script/app2.py", line 132, in report
ws.write(rowNum, colNum, row)
File "/usr/local/lib/python2.7/dist-packages/xlwt/Worksheet.py", line 1088, in write
self.row(r).write(c, label, style)
File "/usr/local/lib/python2.7/dist-packages/xlwt/Row.py", line 252, in write
self.__rich_text_helper(col, label, style, style_index)
File "/usr/local/lib/python2.7/dist-packages/xlwt/Row.py", line 278, in __rich_text_helper
raise Exception ("Unexpected data type %r" % type(data))
Exception: Unexpected data type <type 'datetime.datetime'>
I notice there's error with datetime format data from the SQL data while trying to export to XLS. Is there a way to export datetime data from SQL to XLS? Already trying several solutions from stackoverflow but to no avail
Related
'''
features = [gender, SeniorCitizen, Partner, Dependents, Tenure, PhoneService, MultipleLines, OnlineSecurity, OnlineBackup,
DeviceProtection, TechSupport, StreamingTV, StreamingMovies, PaperlessBilling, MonthlyCharges, TotalCharges,
InternetService_Fiberoptic, InternetService_No, Contract_Oneyear,Contract_Twoyear,
PaymentMethod_CreditCard, PaymentMethod_ElectronicCheck, PaymentMethod_MailedCheck]
final_features = [np.array(features)]
prediction = model.predict_proba(final_features)
output = prediction[0,1]
# Shap Values
explainer = joblib.load(filename="explainer.bz2")
shap_values = explainer.shap_values(np.array(final_features))
shap_img = io.BytesIO()
shap.force_plot(explainer.expected_value[1], shap_values[1], columns, matplotlib = True, show = False).savefig(shap_img, bbox_inches="tight", format = 'png')
shap_img.seek(0)
shap_url = base64.b64encode(shap_img.getvalue()).decode()
The error is coming about tree ensemble when we try to run the python file.The error says: 'TreeEnsemble' object has no attribute 'model_output'. Please help to correct this error
the traceback is :
File "C:\Users\utrej\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1836, in call
return self.wsgi_app(environ, start_response)
File "C:\Users\utrej\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1820, in wsgi_app
response = self.make_response(self.handle_exception(e))
File "C:\Users\utrej\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1403, in handle_exception
reraise(exc_type, exc_value, tb)
File "C:\Users\utrej\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask_compat.py", line 33, in reraise
raise value
File "C:\Users\utrej\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1817, in wsgi_app
response = self.full_dispatch_request()
File "C:\Users\utrej\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1477, in full_dispatch_request
rv = self.handle_user_exception(e)
File "C:\Users\utrej\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1381, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "C:\Users\utrej\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask_compat.py", line 33, in reraise
raise value
File "C:\Users\utrej\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1475, in full_dispatch_request
rv = self.dispatch_request()
File "C:\Users\utrej\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1461, in dispatch_request
return self.view_functionsrule.endpoint
File "C:\Users\utrej\Desktop####\product dev lab\Customer-Survival-Analysis-and-Churn-Prediction-master\app.py", line 118, in predict
shap_values = explainer.shap_values(np.array(final_features))
File "C:\Users\utrej\AppData\Local\Programs\Python\Python38\Lib\site-packages\shap\explainers\tree.py", line 298, in shap_values
if self.model.model_output == "log_loss":
I have the following class that I am using to pool connections for my application.
import os
import psycopg2
import contextlib
from psycopg2 import pool
from contextlib import contextmanager
class DbHandler:
def __init__(self):
self.reconnect_pool()
def reconnect_pool(self):
self.pgPool = psycopg2.pool.SimpleConnectionPool(1, 128, user=os.getenv("PGUSER"), password=os.getenv("PGPASSWORD"), host=os.getenv("PGHOST"), port=os.getenv("PGPORT"), database=os.getenv("PGDATABASE"))
# Get Cursor
#contextmanager
def get_cursor(self):
if(self.pgPool):
conn = self.pgPool.getconn()
if(conn):
try:
yield conn.cursor()
conn.commit()
finally:
self.pgPool.putconn(conn)
else:
self.reconnect_pool()
# Get Cursor
#contextmanager
def get_connection(self):
if(self.pgPool):
conn = self.pgPool.getconn()
if(conn):
try:
yield conn
finally:
self.pgPool.putconn(conn)
else:
self.reconnect_pool()
# Helper function
def getRows(self, cursor):
rows=[]
row = cursor.fetchone()
while row is not None:
rows.append(row)
row = cursor.fetchone()
return rows
def getPoolSize(self):
if(self.pgPool):
return len(self.pgPool._used)
else:
return -1
I am executing the code in the following manner inside my code by instantiating and calling the cursor.
def Login(self,username,password):
with self.db.get_cursor() as psCursor:
psCursor.execute("select users.username, users.id, string_agg(coalesce(keys.service, ''), ',') from users left join keys on keys.user_id = users.id where users.username = '" + username + "' and users.password='" + password + "'" + "group by (users.username,users.id)")
rows = self.db.getRows(psCursor)
if len(rows) == 0:
return None
if len(rows) > 0:
return rows[0]
But once i stop using the application for a while and try to execute any sql query. I get the error below
2020-09-01T07:11:05.074414377Z File "/usr/local/lib/python3.7/site-packages/gunicorn/workers/gthread.py", line 284, in handle
2020-09-01T07:11:05.074428178Z keepalive = self.handle_request(req, conn)
2020-09-01T07:11:05.074432778Z File "/usr/local/lib/python3.7/site-packages/gunicorn/workers/gthread.py", line 333, in handle_request
2020-09-01T07:11:05.074438078Z respiter = self.wsgi(environ, resp.start_response)
2020-09-01T07:11:05.074442378Z File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2463, in __call__
2020-09-01T07:11:05.074446678Z return self.wsgi_app(environ, start_response)
2020-09-01T07:11:05.074450879Z File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2449, in wsgi_app
2020-09-01T07:11:05.074455079Z response = self.handle_exception(e)
2020-09-01T07:11:05.074459279Z File "/usr/local/lib/python3.7/site-packages/flask_cors/extension.py", line 161, in wrapped_function
2020-09-01T07:11:05.074463479Z return cors_after_request(app.make_response(f(*args, **kwargs)))
2020-09-01T07:11:05.074467779Z File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1866, in handle_exception
2020-09-01T07:11:05.074472080Z reraise(exc_type, exc_value, tb)
2020-09-01T07:11:05.074476280Z File "/usr/local/lib/python3.7/site-packages/flask/_compat.py", line 39, in reraise
2020-09-01T07:11:05.074480580Z raise value
2020-09-01T07:11:05.074484580Z File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2446, in wsgi_app
2020-09-01T07:11:05.074488780Z response = self.full_dispatch_request()
2020-09-01T07:11:05.074492880Z File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1951, in full_dispatch_request
2020-09-01T07:11:05.074497181Z rv = self.handle_user_exception(e)
2020-09-01T07:11:05.074501181Z File "/usr/local/lib/python3.7/site-packages/flask_cors/extension.py", line 161, in wrapped_function
2020-09-01T07:11:05.074505381Z return cors_after_request(app.make_response(f(*args, **kwargs)))
2020-09-01T07:11:05.074509581Z File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1820, in handle_user_exception
2020-09-01T07:11:05.074513781Z reraise(exc_type, exc_value, tb)
2020-09-01T07:11:05.074520082Z File "/usr/local/lib/python3.7/site-packages/flask/_compat.py", line 39, in reraise
2020-09-01T07:11:05.074524382Z raise value
2020-09-01T07:11:05.074528382Z File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1949, in full_dispatch_request
2020-09-01T07:11:05.074532682Z rv = self.dispatch_request()
2020-09-01T07:11:05.074536682Z File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1935, in dispatch_request
2020-09-01T07:11:05.074541083Z return self.view_functions[rule.endpoint](**req.view_args)
2020-09-01T07:11:05.074545183Z File "/whaii/routes/index.py", line 126, in login
2020-09-01T07:11:05.074549383Z user = MatchFunc.Login(str(data.get('username')),str(data.get('password')))
2020-09-01T07:11:05.074556783Z File "/whaii/routes/Whaii_Engine.py", line 148, in Login
2020-09-01T07:11:05.074562384Z psCursor.execute("select users.username, users.id, string_agg(coalesce(keys.service, ''), ',') from users left join keys on keys.user_id = users.id where users.username = '" + username + "' and users.password='" + password + "'" + "group by (users.username,users.id)")
2020-09-01T07:11:05.074567584Z psycopg2.DatabaseError: SSL SYSCALL error: Connection timed out
I am guessing it is the way I am calling the cursor. It would be great if someone could point out how I can reuse the connection pool.
Also, I ran SELECT count(*) FROM pg_stat_activity; and I can see that there are about 26 connections that are active but I am confused as to why the query fails.
I want to try to make a dataframe from pdfreader, with the column name is isi1. but why do I get an error ValueError: DataFrame constructor not properly called !. what should i do to fix this error. help from anyone when needed for this problem.
this is my code
if request.method == 'POST':
f = request.files['file']
f.save(f.filename)
pdfreader = PyPDF2.PdfFileReader(open('C:/Users/Novilia/PycharmProjects/tesaja/' + f.filename, 'rb'))
from pandas import DataFrame
df1 = DataFrame(pdfreader, columns=['isi1'])
#vect = count_vect.transform(['isi1']).toarray()
df1['label1'] = text_clf.predict(df1['isi1'])
df1.append(['label1'])
hasil = (df1.isi1[df1['label1'] == 'positif'])
len(hasil)
hasil_list = hasil.values.tolist()
stringList = ' '.join([str(item) for item in hasil_list])
hasil_ringkas = stringList
return render_template('result.html', ringkasan = hasil_ringkas)
the error is
ValueError
ValueError: DataFrame constructor not properly called!
Traceback (most recent call last)
File "C:\Users\Novilia\PycharmProjects\tesaja\venv\lib\site-packages\flask\app.py", line 2463, in __call__
return self.wsgi_app(environ, start_response)
File "C:\Users\Novilia\PycharmProjects\tesaja\venv\lib\site-packages\flask\app.py", line 2449, in wsgi_app
response = self.handle_exception(e)
File "C:\Users\Novilia\PycharmProjects\tesaja\venv\lib\site-packages\flask\app.py", line 1866, in handle_exception
reraise(exc_type, exc_value, tb)
File "C:\Users\Novilia\PycharmProjects\tesaja\venv\lib\site-packages\flask\_compat.py", line 39, in reraise
raise value
File "C:\Users\Novilia\PycharmProjects\tesaja\venv\lib\site-packages\flask\app.py", line 2446, in wsgi_app
response = self.full_dispatch_request()
File "C:\Users\Novilia\PycharmProjects\tesaja\venv\lib\site-packages\flask\app.py", line 1951, in full_dispatch_request
rv = self.handle_user_exception(e)
File "C:\Users\Novilia\PycharmProjects\tesaja\venv\lib\site-packages\flask\app.py", line 1820, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "C:\Users\Novilia\PycharmProjects\tesaja\venv\lib\site-packages\flask\_compat.py", line 39, in reraise
raise value
File "C:\Users\Novilia\PycharmProjects\tesaja\venv\lib\site-packages\flask\app.py", line 1949, in full_dispatch_request
rv = self.dispatch_request()
File "C:\Users\Novilia\PycharmProjects\tesaja\venv\lib\site-packages\flask\app.py", line 1935, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "C:\Users\Novilia\PycharmProjects\tesaja\app.py", line 79, in summarize
df1 = DataFrame(pdfreader, columns=['isi1'])
File "C:\Users\Novilia\PycharmProjects\tesaja\venv\lib\site-packages\pandas\core\frame.py", line 509, in __init__
raise ValueError("DataFrame constructor not properly called!")
ValueError: DataFrame constructor not properly called!
From my understanding, I guess you need to iterate over the pages and get the text from each page. can you try the following:
import nltk
pdfreader = PyPDF2.PdfFileReader(open('C:/Users/Novilia/PycharmProjects/tesaja/' + f.filename, 'rb'))
page_contents = [sent for page_no in range(pdfreader.getNumPages())
for sent in nltk.sent_tokenize(pdfreader.getPage(page_no).extractText())]
df1 = DataFrame(page_contents, columns=['isi1'])
Kindly let me know if you find any problem with the code.
I want to insert multiple records into Mysql using python.
I have tried something like:
if request.method == 'POST':
req_json = request.get_json(force=True)
dir(req_json)
#sys.exit()
g.cursor.execute("INSERT INTO gmr (version,visitorId,dwellTime,poiId,srId,zoneId,poiProximityConfidence,zoneProximityConfidence,poiPresenceConfidence,zonePresenceConfidence,normalizedTime) VALUES ({0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10})",
(req_json[0]['version'], req_json[0]['visitorId'], req_json[0]['dwellTime'], req_json[0]['poiId'], req_json[0]['srId'], req_json[0]['zoneId'], req_json[0]['poiProximityConfidence'], req_json[0]['zoneProximityConfidence'], req_json[0]['poiPresenceConfidence'], req_json[0]['zonePresenceConfidence'], req_json[0]['normalizedTime']))
Im passing json objects like:
[
{
"dwellTime":"2",
"normalizedTime":"1489574977000",
"poiId":"poi3",
"poiPresenceConfidence":"15",
"poiProximityConfidence":"1",
"srId":"sr3",
"version":"15",
"visitorId":"V123",
"zoneId":"Security",
"zonePresenceConfidence":"40",
"zoneProximityConfidence":"20"
},
{
"dwellTime":"234234",
"normalizedTime":"1489574977000",
"poiId":"poi3",
"poiPresenceConfidence":"15",
"poiProximityConfidence":"1",
"srId":"sr3",
"version":"16",
"visitorId":"V123",
"zoneId":"Security",
"zonePresenceConfidence":"40",
"zoneProximityConfidence":"20"
}
]
I get the error like:
ProgrammingError: not all arguments converted during string formatting
Traceback (most recent call last): File "/root/python-mysql/venv/lib/python2.7/site-packages/flask/app.py", line 1997, in call return self.wsgi_app(environ, start_response)
File "/root/python-mysql/venv/lib/python2.7/site-packages/flask/app.py", line 1985, in wsgi_app response = self.handle_exception(e)
File "/root/python-mysql/venv/lib/python2.7/site-packages/flask/app.py", line 1540, in handle_exception reraise(exc_type, exc_value, tb)
File "/root/python-mysql/venv/lib/python2.7/site-packages/flask/app.py", line 1982, in wsgi_app response = self.full_dispatch_request()
File "/root/python-mysql/venv/lib/python2.7/site-packages/flask/app.py", line 1614, in full_dispatch_request rv = self.handle_user_exception(e)
File "/root/python-mysql/venv/lib/python2.7/site-packages/flask/app.py", line 1517, in handle_user_exception reraise(exc_type, exc_value, tb) File "/root/python-mysql/venv/lib/python2.7/site-packages/flask/app.py", line 1612, in full_dispatch_request rv = self.dispatch_request()
File "/root/python-mysql/venv/lib/python2.7/site-packages/flask/app.py", line 1598, in dispatch_request return self.view_functionsrule.endpoint
File "/root/python-mysql/venv/py_gmr.py", line 56, in add (req_json[0]['version'], req_json[0]['visitorId'], req_json[0]['dwellTime'], req_json[0]['poiId'], req_json[0]['srId'], req_json[0]['zoneId'], req_json[0]['poiProximityConfidence'], req_json[0]['zoneProximityConfidence'], req_json[0]['poiPresenceConfidence'], req_json[0]['zonePresenceConfidence'], req_json[0]['normalizedTime']))
File "/root/python-mysql/venv/lib/python2.7/site-packages/MySQLdb/cursors.py", line 283, in executemany self.rowcount = sum(self.execute(query, arg) for arg in args)
File "/root/python-mysql/venv/lib/python2.7/site-packages/MySQLdb/cursors.py", line 283, in self.rowcount = sum(self.execute(query, arg) for arg in args)
File "/root/python-mysql/venv/lib/python2.7/site-packages/MySQLdb/cursors.py", line 240, in execute self.errorhandler(self, ProgrammingError, str(m))
File "/root/python-mysql/venv/lib/python2.7/site-packages/MySQLdb/connections.py", line 52, in defaulterrorhandler raise errorclass(errorvalue) ProgrammingError: not all arguments converted during string formatting
If I replace {0}, {1} with %s, only a single value gets inserted
You cannot use {} as placeholders in an SQL statement. You must use %s.
Using {} wouldn't fix your "problem" with only inserting a single set of values. That is to be expected, because you are only accessing the first set of values in your JSON. If you want to insert them both, you need to iterate over the values with a for loop and insert one set each time - as you would when trying to do anything with any list.
I am writing SQLAlchemy code that takes a search bar's input and runs a query against my PostgreSQL database. To allow for misspellings and name fragments, I had this code that did the trick:
q = session.query(Detail).filter((Detail.sellers.ilike("%%%s%%" % (name_input)))).all()
Now I am trying to do the same thing, but to check against a list names that has multiple input values. I've accomplished that like so:
q = session.query(Detail).filter((Detail.sellers.in_(names))
That requires exact matches, however. I would like to combine the ilike operator and the in_ operator. Is this possible? I've tried:
q = session.query(Detail).filter((Detail.sellers.ilike.in_(names))
That gives the following traceback and error:
File "/Users/Tom/projects/land-records/lib/python2.7/site-packages/flask/app.py", line 1836, in __call__
return self.wsgi_app(environ, start_response)
File "/Users/Tom/projects/land-records/lib/python2.7/site-packages/flask/app.py", line 1820, in wsgi_app
response = self.make_response(self.handle_exception(e))
File "/Users/Tom/projects/land-records/lib/python2.7/site-packages/flask/app.py", line 1403, in handle_exception
reraise(exc_type, exc_value, tb)
File "/Users/Tom/projects/land-records/lib/python2.7/site-packages/flask/app.py", line 1817, in wsgi_app
response = self.full_dispatch_request()
File "/Users/Tom/projects/land-records/lib/python2.7/site-packages/flask/app.py", line 1477, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/Users/Tom/projects/land-records/lib/python2.7/site-packages/flask/app.py", line 1381, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/Users/Tom/projects/land-records/lib/python2.7/site-packages/flask/app.py", line 1475, in full_dispatch_request
rv = self.dispatch_request()
File "/Users/Tom/projects/land-records/lib/python2.7/site-packages/flask/app.py", line 1461, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/Users/Tom/projects/land-records/repo/scripts/app.py", line 162, in search
return query_db(names)
File "/Users/Tom/projects/land-records/lib/python2.7/site-packages/flask_cache/__init__.py", line 537, in decorated_function
rv = f(*args, **kwargs)
File "/Users/Tom/projects/land-records/repo/scripts/app.py", line 106, in query_db
q = session.query(Detail).filter((Detail.sellers.ilike.in_(names))).all()
AttributeError: 'function' object has no attribute 'in_'
If it's just about the case insensitive 'in_' statement:
from sqlalchemy import func
q = session.query(Detail).filter(
func.lower(Detail.sellers).in_(map(str.lower, names))
)
Otherwise, if you wanted to have multiple 'ilike' statements,
from sqlalchemy import or_
conditions = []
for name in names:
conditions.append(Detail.sellers.ilike('%{}%'.format(name)))
q = session.query(Detail).filter(
or_(*conditions)
)
I haven't tested it, in case it didn't work, I can edit it.