Develop Reference

How to: Spyne authentication?

I don't understand how to setup the user and password for authentication of my soap server.
I found this example:
import logging
import random
import sys
# bcrypt seems to be among the latest consensus around cryptograpic circles on
# storing passwords.
# You need the package from http://code.google.com/p/py-bcrypt/
# You can install it by running easy_install py-bcrypt.
try:
import bcrypt
except ImportError:
print('easy_install --user py-bcrypt to get it.')
raise
from spyne.application import Application
from spyne.decorator import rpc
from spyne.error import ArgumentError
from spyne.model.complex import ComplexModel
from spyne.model.fault import Fault
from spyne.model.primitive import Mandatory
from spyne.model.primitive import String
from spyne.protocol.soap import Soap11
from spyne.server.wsgi import WsgiApplication
from spyne.service import Service
class PublicKeyError(Fault):
__namespace__ = 'spyne.examples.authentication'
def __init__(self, value):
super(PublicKeyError, self).__init__(
faultstring='Value %r not found' % value)
class AuthenticationError(Fault):
__namespace__ = 'spyne.examples.authentication'
def __init__(self, user_name):
# TODO: self.transport.http.resp_code = HTTP_401
super(AuthenticationError, self).__init__(
faultcode='Client.AuthenticationError',
faultstring='Invalid authentication request for %r' % user_name)
class AuthorizationError(Fault):
__namespace__ = 'spyne.examples.authentication'
def __init__(self):
# TODO: self.transport.http.resp_code = HTTP_401
super(AuthorizationError, self).__init__(
faultcode='Client.AuthorizationError',
faultstring='You are not authozied to access this resource.')
class SpyneDict(dict):
def __getitem__(self, key):
try:
return dict.__getitem__(self, key)
except KeyError:
raise PublicKeyError(key)
class RequestHeader(ComplexModel):
__namespace__ = 'spyne.examples.authentication'
session_id = Mandatory.String
user_name = Mandatory.String
class Preferences(ComplexModel):
__namespace__ = 'spyne.examples.authentication'
language = String(max_len=2)
time_zone = String
user_db = {
'neo': bcrypt.hashpw('Wh1teR#bbit', bcrypt.gensalt()),
}
session_db = set()
preferences_db = SpyneDict({
'neo': Preferences(language='en', time_zone='Underground/Zion'),
'smith': Preferences(language='xx', time_zone='Matrix/Core'),
})
class AuthenticationService(Service):
__tns__ = 'spyne.examples.authentication'
#rpc(Mandatory.String, Mandatory.String, _returns=String,
_throws=AuthenticationError)
def authenticate(ctx, user_name, password):
password_hash = user_db.get(user_name, None)
if password_hash is None:
raise AuthenticationError(user_name)
if bcrypt.hashpw(password, password_hash) == password_hash:
session_id = (user_name,
'%x' % random.randint(1 << 124, (1 << 128) - 1))
session_db.add(session_id)
else:
raise AuthenticationError(user_name)
return session_id[1]
class UserService(Service):
__tns__ = 'spyne.examples.authentication'
__in_header__ = RequestHeader
#rpc(Mandatory.String, _throws=PublicKeyError, _returns=Preferences)
def get_preferences(ctx, user_name):
if user_name == 'smith':
raise AuthorizationError()
retval = preferences_db[user_name]
return retval
def _on_method_call(ctx):
if ctx.in_object is None:
raise ArgumentError("RequestHeader is null")
if not (ctx.in_header.user_name, ctx.in_header.session_id) in session_db:
raise AuthenticationError(ctx.in_object.user_name)
UserService.event_manager.add_listener('method_call', _on_method_call)
if __name__ == '__main__':
from spyne.util.wsgi_wrapper import run_twisted
logging.basicConfig(level=logging.DEBUG)
logging.getLogger('spyne.protocol.xml').setLevel(logging.DEBUG)
logging.getLogger('twisted').setLevel(logging.DEBUG)
application = Application([AuthenticationService, UserService],
tns='spyne.examples.authentication',
in_protocol=Soap11(validator='lxml'),
out_protocol=Soap11()
)
twisted_apps = [
(WsgiApplication(application), 'app'),
]
sys.exit(run_twisted(twisted_apps, 8000))
When I run this code it gives me access to files from my soap service directory. But my functions for server doesn't work anymore.
How can I configure the soap service to ask for a login and password and then give me access to the server and all the functions?

Data Tables uploading to local PostgreSQL

I am running a data parser/web scraper with python. The parser then pushes the data (SQL Tables) to postgresql. However, I can't find the tables in pgadmin. This is part of a full stack django webapp, using docker, which I did not create but I am trying to get to run locally. As far as I can tell docker containers are working as intended, and so is the dataparsing script. Since I don't know much about issues like this please let me know if there is anything else I should include
Database connection in python
import psycopg2
import logging
import sys
import os
class DatabaseConnection(object):
def __init__(self, user="postgres", password="1234", host="127.0.0.1", port="5432", database="postgres",
course_table="course_info", prereqs_table="prereqs", antireqs_table="antireqs",
requirements_table="requirements", communications_table="communications",
breadth_table="breadth_table"):
if os.getenv("UWPATH_ENVIRONMENT") is not None and os.getenv("UWPATH_ENVIRONMENT") == "docker":
host = "db"
if os.getenv("DB_PASS") is not None:
password = os.getenv("DB_PASS")
if os.getenv("DB_USER") is not None:
user = os.getenv("DB_USER")
if os.getenv("DB_NAME") is not None:
database = os.getenv("DB_NAME")
if os.getenv("DB_HOST") is not None:
host = os.getenv("DB_HOST")
if os.getenv("DB_PORT") is not None:
port = os.getenv("DB_PORT")
self.connection = psycopg2.connect(user=user, password=password, host=host, port=port, database=database)
self.cursor = self.connection.cursor()
self.course_table = course_table
self.prereqs_table = prereqs_table
self.antireqs_table = antireqs_table
self.requirements_table = requirements_table
self.communications_table = communications_table
self.breadth_table = breadth_table
self.root = self.__Logger()
def __Logger(self):
self.logger = logging.getLogger()
if not len(self.logger.handlers):
self.logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
self.logger.addHandler(handler)
return self.logger
def execute(self, command):
try:
# self.root.info(command)
print(command)
self.cursor.execute(command)
return True
except Exception as e:
print(command)
self.root.error(e)
return False
def commit(self):
if self.connection:
self.connection.commit()
def close(self):
self.connection.close()
def select(self, what, table, condition=""):
"""
SELECT <what> FROM <table> <condition>;
:param what: string
:param table: string
:param condition: string
:return: list
"""
command = "SELECT " + what + " FROM " + table + " " + condition + ";"
self.execute(command)
return self.cursor.fetchall()
Trying to access the backend in browser returns this, which makes me believes the tables don't exist in postgresql
Output in txt file (roughly 300,000 lines, IDK if this is useful in analyzing the problem but thought I'd include it either way)

How to omit custom logging handler on special condition?

I wrote custom handler that puts log messages into MySQL database. I want to use this handler only if I was connected properly using mysql-connector python library. Otherwise I want to omit this handler.
class LogDBHandler(logging.Handler):
"""Customized logging handler that puts logs to the database."""
def __init__(self):
super().__init__()
self.table_name = 'log'
self.counter = 0
self.chunk = []
self.sql_conn, self.sql_cursor = self.connect_to_db()
# create log table if doesn't exist
try:
self.create_table()
except mysql.connector.errors.ProgrammingError:
pass
#staticmethod
def connect_to_db():
"""Connect to MySQL database to perform logging."""
credentials = {
"user": os.environ.get("DARWIN_DB_USER"),
"password": os.environ.get("DARWIN_DB_PASSWORD"),
"host": os.environ.get("DARWIN_DB_HOST", "127.0.0.1"),
"port": os.environ.get("DARWIN_DB_PORT", "3306"),
"database": os.environ.get("DARWIN_DB_NAME"),
}
db = mysql.connector.connect(**credentials)
cursor = db.cursor()
return db, cursor
...
This is logging configuration file, where I store all loggers, handlers etc.
[loggers]
keys=root
[handlers]
keys=root, db
[formatters]
keys=formatter
[logger_root]
level=DEBUG
handlers=root, db
[handler_root]
class=FileHandler
level=DEBUG
formatter=formatter
args=('darwin.log', 'w')
[handler_db]
class=libs.logger.LogDBHandler
level=DEBUG
formatter=formatter
args=()
[formatter_formatter]
format=%(asctime)s - %(name)-12s - %(levelname)-8s - %(message)s
Everything works fine if database credentials are valid. If they are wrong mysql.connector.errors.ProgrammingError exception is raised. Instead of shutting down entire program I would like to omit this custom handler if self.connect_to_db() raises exception . Any ideas how to achieve such thing?
Thanks in advance ;)

Reedit with a working mock:
class LogDBHandler(logging.Handler):
def __init__(self):
super().__init__()
self.sql_conn = None
try:
self.sql_conn, self.sql_cursor = self.connect_to_db()
except:
return None
def emit(self, msg):
pass
logging.handlers.LogDBHandler = LogDBHandler
Main:
import logging.config
from logdbhandler import LogDBHandler
logging.config.fileConfig( 'logconf.yaml', )
logger = logging.getLogger()
for l in logger.handlers:
if isinstance(l, LogDBHandler):
if not l.sql_conn:
logger.removeHandler(l)
print(logger.handlers)

Solution that worked for me. I'm aware it's not elegant, but it works.
class LogDBHandler(logging.Handler):
"""Customized logging handler that puts logs to the database."""
def __init__(self):
super().__init__()
self.table_name = 'log'
self.counter = 0
self.chunk = []
self.sql_conn = self.connect_to_db()
if self.sql_conn:
self.sql_cursor = self.sql_conn.cursor()
# create log table if doesn't exist
self.create_log_table()
else:
print("DB Connection error")
def emit(self, record):
"""Called on each log attempt."""
if self.sql_conn:
timestamp = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(record.created))
self.chunk.append((record.name, record.levelname,
str(record.getMessage()), timestamp))
self.counter += 1
# Insert chunk of data into DB
if self.counter % 1000 == 0:
self.insert_chunk()
def flush(self):
"""
Handler destructor, close all db connection and insert final
records into db.
"""
if self.sql_conn:
# Insert last chunk of data
self.insert_chunk()
# Close connection with DB
self.sql_cursor.close()
self.sql_conn.close()
#staticmethod
def connect_to_db():
"""Connect to MySQL database to perform logging."""
credentials = {
"user": os.environ.get("DARWIN_DB_USER"),
"password": os.environ.get("DARWIN_DB_PASSWORD"),
"host": os.environ.get("DARWIN_DB_HOST", "127.0.0.1"),
"port": os.environ.get("DARWIN_DB_PORT", "3306"),
"database": os.environ.get("DARWIN_DB_NAME"),
}
try:
connection = mysql.connector.connect(**credentials)
return connection
except mysql.connector.errors.ProgrammingError:
return None

May be someone will benefit from the one I have eventually created for myself:
# -*- coding: utf-8 -*-
"""
Copied and modified from https://github.com/onemoretime/mySQLHandler/
"""
import MySQLdb
import logging
import time
class mySQLHandler(logging.Handler):
"""
Logging handler for MySQL db.
"""
check_sql = """SHOW TABLES LIKE '{log_table}';"""
create_sql = """CREATE TABLE IF NOT EXISTS {log_table}(
Created text,
Name text,
LogLevel int,
LogLevelName text,
Message text,
Args text,
Module text,
FuncName text,
LineNo int,
Exception text,
Process int,
Thread text,
ThreadName text
)"""
insert_sql = """INSERT INTO {log_table}(
Created,
Name,
LogLevel,
LogLevelName,
Message,
Args,
Module,
FuncName,
LineNo,
Exception,
Process,
Thread,
ThreadName
)
VALUES (
"{dbtime}",
"{name}",
{levelno},
"{levelname}",
"{msg}",
"{args}",
"{module}",
"{funcName}",
{lineno},
"{exc_text}",
{process},
"{thread}",
"{threadName}"
);
"""
def __init__(self, **kwargs):
"""
Customized logging handler that puts logs to MySQL db.
"""
logging.Handler.__init__(self)
self.host = kwargs['host']
self.port = kwargs['port']
self.dbuser = kwargs['dbuser']
self.dbpassword = kwargs['dbpassword']
self.dbname = kwargs['dbname']
self.log_table = kwargs['log_table']
self.sql_conn, self.sql_cursor = self.connect_to_db()
def connect_to_db(self):
"""
Connect to MySQL database to perform logging.
Create log table if does not exist.
"""
try:
conn=MySQLdb.connect(host=self.host,port=self.port,user=self.dbuser,passwd=self.dbpassword,db=self.dbname)
cur = conn.cursor()
cur.execute(mySQLHandler.check_sql.format(log_table = self.log_table))
conn.commit()
table_exist = cur.fetchone()
if not table_exist:
cur.execute(mySQLHandler.create_sql.format(log_table = self.log_table))
conn.commit()
return conn, cur
except Exception: # ignoring connection and table creation exceptions as this handler meant to be used with application db
return None, None
def flush(self):
"""
Override to implement custom flushing behaviour for MySQLdb connection.
"""
if self.sql_conn:
self.sql_cursor.close()
self.sql_conn.close()
def formatDBTime(self, record):
"""
Time formatter.
"""
record.dbtime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(record.created))
def emit(self, record):
"""
Emit a record to MySQL db.
Format the record and send it to the specified database.
"""
if self.sql_conn:
try:
self.format(record)
self.formatDBTime(record)
record.exc_text = logging._defaultFormatter.formatException(record.exc_info).replace('"', "'").replace('\n','').replace('\r','') if record.exc_info else ""
if isinstance(record.msg, str): record.msg = record.msg.replace("'", "''")
sql_stmt = mySQLHandler.insert_sql.format(**record.__dict__, log_table = self.log_table)
self.sql_cursor.execute(sql_stmt)
self.sql_conn.commit()
except Exception:
self.sql_conn.rollback()
self.handleError(record)
logging.conf looks like this:
[handler_mySQLHandler]
#class=logging.handlers.mySQLHandler
class=mySQLHandler.mySQLHandler
kwargs={"host":"hostname", "port":3306, "dbuser":"dubber", "dbpassword": "password", "dbname": "dbname", "log_table":"syslog"}

Python uncaught exception when inserting data into mysql database

I have a python AWS lambda function that takes JSON records, checks them to see if they have required keys, and then inserts into a MySQL db (AWS RDS Aurora). The function gets invoked whenever a new record comes into the stream def handler.
At the moment, Lambda is reporting some errors, but when I look at cloudwatch logs I don't see any errors, which leads me to believe that maybe I'm not handling or catching the exception. Can anyone tell me where the issue might be?
from __future__ import print_function
import base64
import json
import pymysql
RDS_HOST = 'host'
DB_USER = 'dummy_user'
DB_PASSWORD = 'password1234'
DB_NAME = 'crazy_name'
DB_TABLE = 'wow_table'
class MYSQL(object):
'''
This a wrapper Class for PyMySQL
'''
CONNECTION_TIMEOUT = 30
def __init__(self, host, user, password, database, table):
self.host = host
self.user = user
self.password = password
self.database = database
self.table = table
self.connection = self.connect()
def connect(self):
'''
Connects to MySQL instance
'''
try:
connection = pymysql.connect(
host=self.host,
user=self.user,
password=self.password,
db=self.database,
connect_timeout=self.CONNECTION_TIMEOUT
)
return connection
except Exception as ex:
print(ex)
print("ERROR: Unexpected error: Could not connect to AuroraDB instance")
def execute(self, account_id, external_ref_id, timestamp):
'''
Executes command given a MySQL connection
'''
with self.connection.cursor() as cursor:
sql = ('INSERT INTO ' +
self.database +
'.' +
self.table +
'(`account_id`, `external_reference_id`, `registration`, `c_name`, `c_id`, `create_date`)' +
' VALUES (%s, %s, DATE_FORMAT(STR_TO_DATE(%s,"%%Y-%%M-%%d %%H:%%i:%%s"),"%%Y-%%m-%%d %%H:%%i:%%s"), %s, %s, current_timestamp())' +
' ON DUPLICATE KEY UPDATE create_date = VALUES(create_date)')
cursor.execute(sql, (
account_id,
external_ref_id,
timestamp,
'bingo',
300)
)
self.connection.commit()
def close_connection(self):
'''
Closes connection to MySQL
'''
self.connection.close()
def get_data_from_kinesis_object(obj):
'''
Retrieves data from kinesis event
'''
return obj['kinesis']['data']
def decode_data(data):
'''
Decodes record via base64
'''
return base64.b64decode(data)
def split_records_into_record(records):
'''
Splits a record of records into an array of records
'''
return records.split('\n')
def parse_record(record):
'''
parses record into JSON
'''
if record:
return json.loads(record)
def is_record_valid(record):
'''
Check for keys in event
returns True if they all exist
and False if they dont all exist
'''
return all(key in record for key in (
'eventName',
'sourceType',
'AccountId',
'Timestamp',
'ExternalReferenceId'
))
def handler(event, context):
"""
This function inserts data into Aurora RDS instance
"""
mysql = MYSQL(RDS_HOST, DB_USER, DB_PASSWORD, DB_NAME, DB_TABLE)
for obj in event['Records']:
records = decode_data(get_data_from_kinesis_object(obj))
split_records = split_records_into_record(records)
for record in split_records:
parsed_record = parse_record(record)
if is_record_valid(parsed_record):
mysql.execute(
parsed_record['AccountId'],
parsed_record['ExternalReferenceId'],
str(parsed_record['Timestamp'])
)
mysql.close_connection()

Broken Pipe Error on python script with squid

I am running a simple python script to log the accessed url using squid url_rewriter_program.
However every time it runs, rewriter crashes with broken pipe error at sys.stdout.flush().
Please suggest a specific solution.
python code is:
import sys
import os
import io
line = sys.stdin.readline()
fo=open("/home/linux/Desktop/foo1.txt","a")
fo.write(line)
fo.close()
sys.stdout.write("\n")
sys.stdout.flush()

This is a squid redirector file, written on python, can you get, or compare with your script:
Redirector:
#!/usr/bin/env python
# -*- coding: iso-8859-1 -*-
#-----------------------------------------------------------------------------
# Name: redirector_master.py
# Purpose: SiSCont checker cuote
#
# Author: Ernesto Licea Martin
#
# Created: 2011/11/24
# RCS-ID: $Id: redirector_master.py $
# Copyright: (c) 2011
# Licence: GPL
#-----------------------------------------------------------------------------
import sys, string, libSiSCont
__name__= "redirector_master"
query="SELECT accounts_proxyquotatype.name, accounts_proxyaccount.proxy_quota, accounts_proxyaccount.proxy_quota_extra, accounts_proxyaccount.proxy_active, accounts_proxyaccounttype.name FROM accounts_proxyaccount INNER JOIN accounts_proxyaccounttype INNER JOIN accounts_proxyquotatype ON (accounts_proxyaccount.proxy_quota_type_id = accounts_proxyquotatype.id) AND (accounts_proxyaccount.proxy_account_type_id = accounts_proxyaccounttype.id) WHERE accounts_proxyaccount.proxy_username=%s"
class RedirMaster:
def __init__(self):
obj = libSiSCont.ParceConf()
obj.parcecfgfile()
self.__listModules = obj.getModList()
self.__redirDicc = obj.getRedirectURL()
self.__penalURL = obj.getPenalizedURL()
self.__confDicc = obj.getConfParam()
self.__dbDicc = obj.getDBDicc()
self.__proxyDicc = obj.getProxyCacheParam()
self.__dbParam = []
def getProxyTypes(self):
db=libSiSCont.connectDB(dbDicc=self.__dbDicc)
c=db.cursor()
c.execute("SELECT accounts_proxy")
def run(self):
modules=[]
for mod in self.__listModules:
try:
m=__import__(mod)
modules.append(m)
except Exception, e:
libSiSCont.reportLogs("%s.run" %__name__, 'Unable to load redirector module %s; the error was: %s' % (mod,str(e)))
if len(modules) == 0:
libSiSCont.reportLogs("%s.run" %__name__, 'No usable redirector module found; switching to trasparent behavour')
while 1:
try:
data_redir=raw_input()
data_redir=data_redir.split()
url,ip_host,user,method,urlgroup = data_redir[0:5]
ip=ip_host.split("/")[0]
host_name=ip_host.split("/")[1]
uri = url
mode=""
#Don't check cache access
if string.find(url,"cache_object") == 0:
sys.stdout.write("%s%s\n" %(mode,uri))
sys.stdout.flush()
continue
db=libSiSCont.connectDB(dbDicc=self.__dbDicc)
c=db.cursor()
c.execute(query,user)
cuote_type,cuote, ext_cuote, active, acc_type = c.fetchall()[0]
self.__dbParam=[cuote_type,int(cuote), int(ext_cuote), active, acc_type]
for module in modules:
try:
uri = module.redir(url = url, ip = ip, host_name = host_name, user = user, method = method, urlgroup = urlgroup, redirDicc = self.__redirDicc, penalURL = self.__penalURL, confDicc = self.__confDicc, proxyDicc = self.__proxyDicc, dbParam = self.__dbParam)
except Exception, e:
libSiSCont.reportLogs("%s.run" %__name__, 'Error while running module: %s -- The error was: %s' % (module,str(e)))
if uri != url:
mode = "301:"
break
sys.stdout.write("%s%s\n" %(mode,uri))
sys.stdout.flush()
except Exception, e:
if not string.find('%s' % e,'EOF') >= 0:
sys.stdout.write('%s\n' % uri)
sys.stdout.flush()
libSiSCont.reportLogs("%s.run" %__name__, '%s: data received from parent: %s' % (str(e),string.join(data_redir)))
else:
sys.exit()
obj=RedirMaster()
obj.run()
Helper:
#!/usr/bin/env python
import sys, syslog, libSiSCont, string,crypt
__name__ = "Helper"
query = "SELECT accounts_proxyaccount.proxy_username FROM accounts_proxyaccount WHERE accounts_proxyaccount.proxy_username=%s AND accounts_proxyaccount.proxy_password=%s"
class BasicAuth:
def __init__(self):
obj = libSiSCont.ParceConf()
obj.parcecfgfile()
self.__dbDicc = obj.getDBDicc()
def run(self):
while 1:
try:
user_pass = string.split(raw_input())
user = user_pass[0].strip("\n")
passwd = user_pass[1].strip("\n")
crypt_passwd = crypt.crypt(passwd,user)
db = libSiSCont.connectDB(self.__dbDicc)
c = db.cursor()
c.execute(query,(user,crypt_passwd))
if c.fetchone() == None:
libSiSCont.reportLogs('%s.run' %__name__,'User Authentication Fail, user = %s password= %s, Access Denied' %(user,passwd) )
sys.stdout.write("ERR\n")
sys.stdout.flush()
else:
libSiSCont.reportLogs('%s.run' %__name__, 'User Authentication Success, user = %s, Access Granted' %user)
sys.stdout.write("OK\n")
sys.stdout.flush()
except Exception, e:
if not string.find("%s" %e, "EOF") >= 0:
sys.stdout.write("ERR\n")
sys.stdout.flush()
libSiSCont.reportLogs('%s.run' %__name__, 'Authenticator error, user will navigate without authentication: %s' %str(e))
else:
sys.exit()
obj = BasicAuth()
obj.run()
I hope help you ;-)