Develop Reference

Issue implementing the python code around cx_Oracle library

I am getting an error while implementing the below code:
'''
from pandas.core.frame import DataFrame
import cx_Oracle
import pandas as pd
import sys
class IFTDataCore:
def __init__(self, accountCode):
i = 0
all_Procedures = []
dns_tns = cx_Oracle.makedsn("gbhenora06vd.corp.amvescap.net", "1525", "INVU")
db=cx_Oracle.connect("CORP-SVC-IFT", "C$Rp$vc1ftUat",dns_tns)
cursor = db.cursor()
cursor.execute("select procedure_name from all_procedures where object_name = 'PK_IVZ_IFT_EXTRACT' ")
rows = cursor.fetchall()
procedureName = ['PK_IVZ_IFT_EXTRACT.'+str(list(rows[indexRow])[0]) for indexRow in range(0,len(list(rows)))]
l_cur = cursor.var(cx_Oracle.CURSOR)
while i < len(procedureName):
if procedureName[i] == 'PK_IVZ_IFT_EXTRACT.SP_IVZ_IFT_EXTRACT_ACCOUNTS':
ret_cursor = cursor.callproc(procedureName[i],(l_cur,))
dfx = pd.DataFrame(ret_cursor[0])
all_Procedures.append(dfx)
else:
ret_cursor = cursor.callproc(procedureName[i],(l_cur,accountCode))
dfx = pd.DataFrame(ret_cursor[0])
all_Procedures.append(dfx)
i += 1
self.all_Procedures = all_Procedures
cursor.close()
db.close()
#property
def getallProcedures(self):
return self.all_Procedures
if __name__ == '__main__':
Procedures = []
all_Proc = IFTDataCore('TOUHI')
Procedures = all_Proc.getallProcedures()
print(Procedures[0])
PS: The code works fine if I do not put the logic in init and call the def logic directly in code. Please let me know the possible reason why when class initialization is done in main, the definition starts throwing error.

The solution works fine now as per the below code:
from pandas.core.frame import DataFrame
import cx_Oracle
import pandas as pd
import sys
import json
from pathlib import Path
import os
class IFTDataCore:
def __init__(self):
try:
db = cx_Oracle.connect('invest/invest#INVD.WORLD')
cursor = db.cursor()
cursor.execute("select procedure_name from all_procedures where object_name = 'PK_IVZ_IFT_EXTRACT' ")
rows = cursor.fetchall()
procedureName = ['PK_IVZ_IFT_EXTRACT.'+str(list(rows[indexRow])[0]) for indexRow in range(0,len(list(rows))-1)]
# To convert Accounts procedure to JSON format
l_cur_Account = cursor.var(cx_Oracle.CURSOR)
ret_cursor_Account = cursor.callproc(procedureName[1],(l_cur_Account,))
self.dfx_Account = pd.DataFrame(ret_cursor_Account[0])
self.dfx_Account.columns = ['fundCode', 'fundName', 'legalEntitiyIdentifier','isin']
result_Account = self.dfx_Account.to_json(orient='records')
except BaseException as e:
raise
def lambda_handler(event, context):
positional_data = IFTDataCore()
df_acct = positional_data.dfx_Account
df_acct=df_acct.fillna("")
Json=df_acct.to_json(orient='records')
lambda_response = __lambda_response__('200', Json)
return lambda_response
def __lambda_response__(status_code, response_body):
return {
'statusCode': status_code,
'headers': {
'Access-Control-Allow-Headers': 'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token',
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'OPTIONS,GET'
},
'body': response_body
}

Selecting a Design Pattern in Python

The below code I modified from Multiple Inheritance to Facade Data Pattern , however it is still breaking the concept of the Facade Data pattern as my subsystems(ES) are sharing amongst themselves .
All the ES are working on a Structured data stream and enriching the data together , they all are run in Async and then gathered into a list (using async gather) . I wanted to know which data pattern suits this
Used case ,
Where I can keep adding ES as per my requirement.
Each ES can share data amongst itself like dictonary .
And I if I have to add new functionality I follow "Single Responsibility Principle"
Multiple Inheritance
import os
import asyncio
import psycopg2
import websockets
from datetime import datetime
from websockets.extensions import permessage_deflate
from structure import Structure
import sys
sys.path.append('..')
from event_stations.es3a import ES3A
from event_stations.es3b import ES3B
from event_stations.es3c import ES3C
from event_stations.es3d import ES3D
from event_stations.es1a import ES1A
from event_stations.es1b import ES1B
from event_stations.es2a import ES2A
from event_stations.es2b import ES2B
class FR_Server(ES1A, ES2A, ES2B, ES3A, ES3B, ES3C, ES3D, Structure):
unique_id = "100"
event_format_list = []
fr_config_table = 'detail_event.app_fer_config'
psql_db = None
def __init__(self):
print("Receiver Called INIT")
self.psql_db = self.connect_to_psql()
self._get_metadata()
super(FR_Server, self).__init__()
# self.start_receiver(self.mapped_dict)
def connect_to_psql(self):
db = psycopg2.connect("dbname=trimble user=postgres password=admin")
return db
def _get_parent_classname(self):
_parent_classes = []
_cls_obj = eval("FR_Server")
for parent in _cls_obj.__bases__:
_parent_classes.append(parent.__name__)
return _parent_classes
def _get_metadata(self):
_parents = self._get_parent_classname()
print(_parents, "pppp")
cur = self.psql_db.cursor()
cur.execute(f"Select * from {self.fr_config_table}")
results = cur.fetchall()
for result in results:
event_station_id = result[0]
if event_station_id in _parents:
event_station_classname = eval(event_station_id)
setattr(event_station_classname, "cache_table_name", result[1])
setattr(event_station_classname, "ignite_port", result[2])
setattr(event_station_classname, "ignite_host", result[3])
def get_port(self):
return os.getenv('WS_PORT', '10011')
def get_host(self):
return os.getenv('WS_HOST', 'localhost')
async def start(self):
return await websockets.serve(self.handler, self.get_host(), self.get_port(), ping_interval=None, max_size=None,
max_queue=None, close_timeout=None, extensions=[
permessage_deflate.ServerPerMessageDeflateFactory(
server_max_window_bits=11,
client_max_window_bits=11,
compress_settings={'memLevel': 4},
),
])
def generate_event_id(self, index):
return "".join(['%02d%02d%d%d%d%d%d' % (datetime.now().day, datetime.now().month, datetime.now().year,
datetime.now().hour,datetime.now().minute, datetime.now().second,
datetime.now().microsecond), self.unique_id,index])
async def handler(self, websocket, path):
async with websockets.connect('ws://localhost:10015', ping_interval=None, max_size=None,
max_queue=None, close_timeout=None,
extensions=[permessage_deflate.ClientPerMessageDeflateFactory(
server_max_window_bits=11,
client_max_window_bits=11,
compress_settings={'memLevel': 4},
),
]) as websocket_rb:
async for row in websocket:
lst_row = row.decode().split(",")
uid = self.generate_event_id(lst_row[0])
lst_row = [uid] + lst_row
results = await asyncio.gather(self.enrich_column_es3a_dict(lst_row[1]),
self.enrich_column_es3b_dict(lst_row[1]),
self.enrich_column_es3c_dict(lst_row[1]),
self.enrich_column_es3d_dict(lst_row[1]))
await websocket_rb.send(str(lst_row + results).encode())
def start_receiver(self, mapped_list):
self.event_format_list = mapped_list
asyncio.get_event_loop().run_until_complete(self.start())
asyncio.get_event_loop().run_forever()
Facade Data pattern :
from __future__ import annotations
from event_stations.es1a import ES1A
from event_stations.es2a import ES2A
from event_stations.es2b import ES2B
import psycopg2
class Foundation_Facade(object):
psql_db = None
client = None
def __init__(self, es1a: ES1A, es2a: ES2A) -> None:
self._es1a = es1a or ES1A()
self._es2a = es2a or ES2A()
def operation(self):
print("Called")
results = []
self.psql_db = self._es1a._connect_psql()
self._es1a._get_metadata(self.psql_db.cursor())
self.client = self._es1a.connect_ignite_client(self._es1a.ignite_host, self._es1a.ignite_port)
self._es2a._get_metadata(self.psql_db.cursor())
self._es2a.put_data(self.client)
self._es2b._get_metadata(self.psql_db.cursor())
self._es2b.put_data(self.client)
print(self._es2b.static_df.head())
# results.append(self._es1a._get_metadata())
return results
if __name__ == '__main__':
es1a = ES1A()
es2a = ES2A()
es2b = ES2B()
facade = Foundation_Facade(es1a, es2a)
from fr_server_1 import Server
Server(facade)

CherryPy WS is not returning string in UTF-8

I'm trying to build a REST Web Service with CherryPy and Python. It works, but when I access it through Chrome, it's not displaying in UTF-8.
This web service queries a MongoDB and then gives the results in a list of dictionaries. I did a print(ticketME) and it's showing the right characters:
But when it displays in Chrome, is not displaying right(and I'm also realizing that "solucion" or "problema" are not showing...):
As you can see in the code, I set the charset to UTF-8:
import cherrypy
import pymongo
import urllib
import pyodbc
import mysql.connector
from datetime import datetime
import time
import sys
import numpy as np
class HelloWorld(object):
#cherrypy.expose
#cherrypy.tools.json_out()
def index(self):
password = * password *
myclient = pymongo.MongoClient(*mongoDB connection string*)
mydb = myclient["moica2"]
mycol = mydb["moicaTickets"]
myquery = *mongoDB aggregate query*
mydoc = mycol.aggregate(myquery)
mydb = None
myclient.close()
mycol = None
resultadoTicketsME = []
for x in mydoc:
try:
asunto = x['pAsunto']
nrotkt = x['pTicket']
estado = x['pEstado']
fechaCreacion = x['pFechaCreacion']
fechaCierre = x['pFechaCierre']
nodoCRM = x['pNodoCRM']
nodoCMTS = x['pNodoCMTS']
if ('COMPLETO' in nodoCMTS):
nodoCMTS = "Completo"
RTs = x['pRTs']
notas = x['pNotas']
asuntoCierre = x['pAsuntoCierre']
estadoEtaClick = x['pEstadoEtaClick']
afectacion = x['pAfectacion']
problema = x['pProblema']
solucion = x['pSolucion']
arbolCreacion = x['pElArbolCreacion']
arbolActual = x['pElarbolActual']
idFuente = int(x['pFuente']['idFuente'])
ticketME = {
'nrotkt': nrotkt,
'asunto': asunto,
'estado': estado,
'fechaCreacion': fechaCreacion,
'fechaCierre': fechaCierre,
'nodoCRM': nodoCRM,
'nodoCMTS': nodoCMTS,
'RTs': RTs,
'notas': notas,
'asuntoCierre': asuntoCierre,
'estadoEtaClick': estadoEtaClick,
'afectacion': afectacion,
'problema': problema,
'solucion': solucion,
'arbolCreacion': arbolCreacion,
'arbolActual': arbolActual,
'idFuente': idFuente
}
print(ticketME)
resultadoTicketsME.append(ticketME)
except:
lf = open("error.log", "a+")
lf.write("MOICA2FUENTESME %s : No se pudo insertar el tkt %s\n" % (datetime.now().strftime("%Y-%m-%d %H:%M:%S"),x['pTicket']))
lf.close()
cherrypy.response.headers['Content-Type'] = "text/html;charset=utf-8"
return resultadoTicketsME
USERS = {'ngabioud': 'password'}
def validate_password(realm, username, password):
if username in USERS and USERS[username] == password:
return True
return False
cherrypy.config.update({'tools.encode.on': True,
'tools.encode.encoding': 'utf-8',
'tools.decode.on': True,
'tools.auth_basic.on': True,
'tools.auth_basic.realm': 'localhost',
'tools.auth_basic.checkpassword': validate_password,
'tools.auth_basic.accept_charset': 'UTF-8',
})
cherrypy.quickstart(HelloWorld())
Is there anything else I could try?
Thank you,
Best regards

As stated by snakecharmerb in the comment, it was a Chrome representing issue. I did a .php seting encoding to utf-8 and it showed correctly.

key error while trying to pass data as a parameter using a json file

code is given below
import time
import unittest
import logging as log
from loggenerator import set_log_params,move_latest_log_to_persistent_file
from parse_test_json import get_test_params
from scrapping import SC
class ScrapeTest(unittest.TestCase):
def setup(self):
self.start_time=time.time()
def teardown(self):
self.end_time=time.time()
def test_SC_scrape(self):
try:
test_name="test scrape"
set_log_params(log_file=test_name,level=log.INFO)
log.info("step1:set all test params")
test_params = get_test_params(self.__class__.__name__, test_name=test_name)
log.info("step 2:set")
ik=SC()
log.info("step3:calling scrapper for")
log.debug(ik.parseURL(party_name=test_params["party_name"],start_date=test_params["start_date"],end_date=test_params["end_date"]))
except Exception as e:
raise e
move_latest_log_to_persistent_file(log_file=test_name)
####
import json, os
from builtins import *
def get_test_params(test_class_name = None, test_name = None):
dir_path = os.path.dirname(os.path.realpath(__file__))
file_path = "/test_param_jsons/" + test_class_name + "params.json"
json_file = dir_path + file_path
with open(json_file) as test_data:
test_json = json.load(test_data)
return test_json[test_class_name][test_name]
this function is raising error key error.

this should work as long as you have a json file available at: <SCRIPT_PATH>/test_param_jsons/MyClass_params.json
Also, in order to avoid KeyError you'll need to ensure that your input json file contains key: value, test_class_name : test_name
import json, os
from builtins import *
class MyClass:
def get_test_params (self, test_class_name = None, test_name = None):
with open(os.path.join(os.path.dirname(__file__),"test_param_jsons\\params.json"), 'r') as test_data:
test_json = json.load (test_data)
try:
return test_json[test_class_name][test_name]
except KeyError as e:
print ('KeyError: {}'.format (e))
def mymain(self, test_name):
''' mymain is defined to accomodate your requirement to use __class__ as a parameter '''
test_params = self.get_test_params (self.__class__.__name__, test_name = test_name)
return test_params
if __name__ == '__main__':
test_name = 'sth'
myclass = MyClass ()
result = myclass.mymain (test_name)
print (result)

Python : No module named 'flask'

So i've been stuck on this problem for a while now. For one file I have it is working and it is allowing me to use the flask framework. with this code.
from flask import Flask, render_template
from flask import request
from flask import *
from datetime import datetime
from functools import wraps
import time
import csv
app = Flask(__name__)
app.secret_key ='lukey'
#displays index page
#app.route('/')
def home():
return render_template('index.html')
#displays welcome page
#app.route('/welcome')
def welcome():
return render_template('welcome.html')
#allows user to login
#app.route('/log', methods=['GET','POST'])
def log():
error = None
if request.method == "POST":
if request.form['user'] != 'admin' or request.form['pass'] != 'admin':
error = "Invalid credentials"
else:
session['logged_in'] = True
return redirect (url_for('welcome'))
return render_template('log.html', error=error)
#allows user to logout
#app.route('/logout')
def logout():
session.pop('logged_in', None)
flash('you were logged out')
return redirect (url_for('log'))
#function to check if admin is logged in
def login_required(test):
#wraps(test)
def wrap(*args, **kwargs):
if 'logged_in' in session:
return test(*args, **kwargs)
else:
flash('you need to login before using admin tools')
return redirect(url_for('log'))
return wrap
#Displays map
#app.route('/map')
def map():
return render_template('map.html')
#Displays gallery
#app.route('/gallery')
def gallery():
return render_template('gallery.html')
#Allows users to view previous bookings
#app.route('/bookings', methods = ['GET'])
def bookings():
bookingsFile ='static\\bookings.csv'
data = readFile(bookingsFile)
return render_template('bookings.html', data=data)
#Allows user to request for a booking
#app.route('/addBookings', methods = ['POST'])
def addBookings():
bookingsFile = 'static\\bookings.csv'
data = readFile(bookingsFile)
bookingName = request.form[('name')]
bookingEmail = request.form[('email')]
bookingDate= request.form[('date')]
#Converts the date string to unix timestamp
bookingDateUnix = time.mktime(datetime.strptime(request.form[('date')], "%Y-%m-%d").timetuple())
numberOfDays = request.form[('days')]
#calculates the end date in unix form
endDateUnix = int(numberOfDays)*24*60*60+int(bookingDateUnix)
#converts the unix form end date to string
newDate = datetime.fromtimestamp(int(endDateUnix)).strftime('%Y-%m-%d')
#Calculates the price of the users stay
price = int(numberOfDays) * 200
#Will be changed by admin to confirm bookings
hasBeenBooked = 'Awaiting confirmation'
bookingsFile ='static\\bookings.csv'
for row in data:
prevBookingDateUnix = row[7]
prevEndDateUnix = row[8]
#Testing no double bookings
if row[2] == bookingDate or row[6] == newDate:
flash('This time has already been allocated')
return redirect(url_for('bookings'))
#Testing there are no crossover points
elif float(prevBookingDateUnix) < bookingDateUnix and float(prevEndDateUnix) < bookingDateUnix and bookingDateUnix < endDateUnix:
flash('valid input')
else:
flash('invalid input')
return redirect(url_for('bookings'))
#parameters parsed from input
newEntry =[bookingName, bookingEmail, bookingDate, numberOfDays, hasBeenBooked, price, newDate, bookingDateUnix, endDateUnix]
data.append(newEntry)
writeFile(data, bookingsFile)
return render_template('bookings.html', data=data)
#allows viewing of comments in csv file
#app.route('/comments', methods = ['GET'])
def comments():
commentsFile = 'static\\comments.csv'
data = readFile(commentsFile)
return render_template('comments.html', data=data)
#adding comments to csv file
#app.route('/addComments', methods = ['POST'])
def addComments():
# add an entry to the data
#read the data from file
commentsFile = 'static\\comments.csv'
data = readFile(commentsFile)
#add the new entry
commentorsName = request.form[('commentorsName')]
comment = request.form[('comment')]
commentDate = datetime.now().strftime("%Y-%m-%d / %H:%M")
newEntry = [commentorsName, comment, commentDate]
data.append(newEntry)
#save the data to the file
writeFile(data, commentsFile)
return render_template('comments.html', data=data)
#Ensures the administrator is logged in before comments are deleted
#app.route('/deleteComments', methods = ['POST'])
#login_required
def deleteComments():
f = open('static\\comments.csv', 'w')
f.truncate()
f.close()
return render_template('comments.html')
#Ensures the administrator is logged in before bookings are deleted
#app.route('/deleteBookings', methods = ['POST'])
#login_required
def deleteBookings():
f = open('static\\bookings.csv', 'w')
f.truncate()
f.close()
return render_template('bookings.html')
def readFile(aFile):
#read in 'aFile'
with open(aFile, 'r') as inFile:
reader = csv.reader(inFile)
data = [row for row in reader]
return data
def writeFile(aList, aFile):
#write 'aList' to 'aFile'
with open(aFile, 'w', newline='') as outFile:
writer = csv.writer(outFile)
writer.writerows(aList)
return
if __name__ == '__main__':
app.run(debug = True)
But with this code it throws the error. No module named 'flask'
#!/usr/bin/python3.4
#
# Small script to show PostgreSQL and Pyscopg together
#
from flask import Flask, render_template
from flask import request
from flask import *
from datetime import datetime
from functools import wraps
import time
import csv
import psycopg2
app = Flask(__name__)
app.secret_key ='lukey'
def getConn():
connStr=("dbname='test' user='lukey' password='lukey'")
conn=psycopg2.connect(connStr)
return conn
#app.route('/')
def home():
return render_template(index.html)
#app.route('/displayStudent', methods =['GET'])
def displayStudent():
residence = request.args['residence']
try:
conn = None
conn = getConn()
cur = conn.cursor()
cur.execute('SET search_path to public')
cur.execute('SELECT stu_id,student.name,course.name,home_town FROM student,\
course WHERE course = course_id AND student.residence = %s',[residence])
rows = cur.fetchall()
if rows:
return render_template('stu.html', rows = rows, residence = residence)
else:
return render_template('index.html', msg1='no data found')
except Exception as e:
return render_template('index.html', msg1='No data found', error1 = e)
finally:
if conn:
conn.close()
##app.route('/addStudent, methods =['GET','POST']')
#def addStudent():
if __name__ == '__main__':
app.run(debug = True)
I feel like the problem is going to be something to do with the versions of python/flask/pip i'm using. Any ideas thank you.

Your Python version is 2.X.
Take a look at this question and its answers.
Your best bet is to use virtualenv, as it makes handling package versions very simple. The accepted answer includes the proper command prompt commands if you want to use Python 3 for this app:
virtualenv -p C:\Python34\python.exe py3env
py3env\Scripts\activate
pip install package-name

I would recommend using Anaconda. Download, install, then run:
conda install flask
And you're done.