Develop Reference

Class architecture to represent a single query with connection and processing attribures

I need to build a piece of software that is able to pull data from different data sources, each data source is represented by a (server, db_name) and for each data sources I need to run a specific query (parametrized by 1,2 fields).
This is the basic structure I came up with, I am not convinced that this is the best way to implement this, especially the get_data() function inside the base class.
My question is then, do you think there's a better way of implementing this logic ?
class BrandedCube(Cube):
def __init__(self, measure:str="ALL", cache_path=""):
super().__init__(measure=measure)
self.server = 'SERVER_NAME'
self.db_name = 'DB_NAME'
self.source_name = ""
self.account_name_hierarchy = ["Account Name"]
self.cache_path = cache_path
self.query = """
EVALUATE
SUMMARIZECOLUMNS(
# STATIC LIST OF COLUMNS,
# STATIC LIST OF FILTERS,
# SOME PARAMETRIC FILTERS
KEEPFILTERS( TREATAS( {{"{qtr}"}}, 'Calendar'[Quarter] )),
"{measure}", [{measure}]
)
"""
def preprocess(self, df:pd.DataFrame) -> pd.DataFrame:
# Eaxample of processing steps
df.rename(columns = {
'Calendar[qtr_sort]': 'Quarter'},
inplace=True)
for col in self.measure_list:
df[col] = df[col].astype(str).astype(float).fillna(0)
df.reset_index(drop=True, inplace=True)
return df
class Cube:
def __init__(self, measure:str):
self.server = ""
self.db_name = ""
# Query to pull data
self.query = """"""
# Path to save output data
self.cache_path = ""
# Quarter format string
self.quarter_format = self.generate_std_format()
# Use or not the MAAD library to pull data
self.maad_library = False
self.maad_library_dbtype = "ssas" # ssas, sql
self.measure_list = self.generate_measure_list(measure)
self.measure = measure
self.id_col: str = ""
self.kpi_dict = {}
self.kpi_list = []
def get_data(self, qtr:str, measure:str) -> pd.DataFrame:
""" Query the cube with the query stored in the class.
The query is formatted with the quarter parameter.
1. maad_library = True : Use the maad library to pull the data -> require a MDX query string
2. maad_library = False: Use Luis library to pull data -> require DAX query
"""
if self.maad_library:
mdx_query = self.query.format(qtr=qtr, measure=measure)
df = maad.get_data(mdx_query, db=self.db_name, dbtype=self.maad_library_dbtype)
else:
format_query = self.query.format(qtr=qtr, measure=measure)
conn = create_connection(self.server, self.db_name)
df = ssas_api.get_DAX(conn, format_query)
return df

When using Q objects in django queryset for OR condition error occured

I want to use filter with OR condition in django.
For this i want to use Q objects.
My code is this
def product(request):
try:
proTitle = request.GET.get('title')
ProDescription = request.GET.get('description')
except:
pass
list0 = []
result = Product.objects.filter(Q(title__contains=proTitle ) | Q(description__contains=ProDescription ) )
for res in result:
list0.append(res.project_id)
data ={'title result':list0}
return HttpResponse(json.dumps(data))
return HttpResponse(json.dumps(data), content_type='application/json')
When pass all value that is proTitle,ProDescription
it working fine.
If any value is going to none it encounter a errorCannot use None as a query value`
Why this error occured when i am using OR operator into my queryset
I am also try this
result = Project.objects.filter(title__contains=proTitle) | Project.objects.filter(description__contains=ProDescription )
but same error occured
I am unable to understand actually what is the problem

You may set the defaults of the get method to something other than None, say empty string:
proTitle = request.GET.get('title', '')
ProDescription = request.GET.get('description', '')
funAria = request.GET.get('funAria', '')
femaleReq = request.GET.get('femaleReq', '')
This is however likely to return all results in the DB when using __contains.
Otherwise you may build the Q functions discarding all None values.
Use this as a guideline: How to dynamically compose an OR query filter in Django?

You can build the Q object in a loop, discarding all values that are none. Note that after consideration, it appears that your except will never be reached because get will simply return None if the Get parameter is not present. You can thus replace the try with an if.
The following code assumes that you simply want the unset values to be ignored in the filter.
(This is untested--let me know if you have any issues.)
attributes = (('title', 'title__contains'),
('description', 'description__contains'),
('funAria', 'functional_area__contains'),
('femaleReq', 'female_candidate'))
query_filter = Q() # initialize
for get_param, kw_arg in attributes:
get_value = request.GET.get(get_param)
if get_value:
query_filter |= Q(**{ kw_arg: get_value })
result = Product.objects.filter(query_filter)

Your error seems to mean that one of your values is None:
def product(request):
try:
proTitle = request.GET.get('title')
ProDescription = request.GET.get('description')
funAria = request.GET.get('funAria')
femaleReq = request.GET.get('femaleReq')
someIntValue = request.GET.get('someIntValue')
except:
pass
allQs = Q()
if proTitle is not None:
allQs |= Q(title__contains=proTitle )
if ProDescription is not None:
allQs |= Q(description__contains=ProDescription )
if funAria is not None:
allQs |= Q(functional_area__contains=funAria )
if someIntValue is not None:
allQs |= Q(some_int_value__gte=someIntValue) # no need to convert someIntValue to an int here as long as you are guaranteed it is unique.
allQs |= Q(female_candidate=femaleReq )
list0 = []
result = Product.objects.filter(allQs)
for res in result:
list0.append(res.project_id)
data ={'title result':list0}
return HttpResponse(json.dumps(data))

Python: how to parse and and add contents to a text file

Hello I have a network in a particular format, i.e. .gdf. However this is a text file in the following format
network:
nodedef>name VARCHAR,label VARCHAR
0,' 0 '
1,' 1 '
2,' 2 '
edgedef>node1 VARCHAR,node2 VARCHAR,weight DOUBLE
0,1,0.2
0,2,0.2
0,3,0.2
0,4,0.333333
where the first part refers to nodes and the second part to edges.
I want to add feature to read the file and add a feature to the nodes and return the following:
network:
nodedef>name VARCHAR,label VARCHAR, att1 VARCHAR
0,' 0 ', 'Paul'
1,' 1 ', 'Jack'
2,' 2 ', 'John'
edgedef>node1 VARCHAR,node2 VARCHAR,weight DOUBLE
0,1,0.2
0,2,0.2
0,3,0.2
0,4,0.333333

Here is some code that does the first half of what you asked for. It will parse the .GDF file and make the information available to you. Adding attributes and writing them is left as an exercise for the reader.
import ast
import collections
import re
def main():
parser = GDFParser()
with open('network.gdf') as file:
parser.read(file)
print(*parser.data, sep='\n')
def pivot(iterable):
columns = []
for row in iterable:
columns.extend([] for _ in range(len(row) - len(columns)))
for column, cell in zip(columns, row):
column.append(cell)
return columns
class GDFParser:
HEADER = re.compile('\w+:')
DEF = re.compile('\w+>\w+ (?:DOUBLE|VARCHAR)(?:,\w+ (?:DOUBLE|VARCHAR))*')
CAST = dict(DOUBLE=float, VARCHAR=str)
def __init__(self):
self.__header = None
self.__type = []
self.__data = []
#property
def header(self):
return self.__header
#property
def data(self):
return tuple(self.__data)
def read(self, file):
for line in file:
self.__read_line(line.strip())
def __read_line(self, line):
if self.HEADER.fullmatch(line):
self.__process_header(line)
elif self.DEF.fullmatch(line):
self.__process_def(line)
else:
self.__process_data(line)
def __process_header(self, line):
if self.header:
raise ValueError('header was previously set')
self.__header = line[:-1]
def __process_def(self, line):
name, fields = line.split('>')
columns, casts = pivot(field.split() for field in fields.split(','))
self.__type.append((collections.namedtuple(name, columns),
tuple(map(self.CAST.__getitem__, casts))))
def __process_data(self, line):
if not self.__type:
raise ValueError('a definition must come before its data')
kind, casts = self.__type[-1]
self.__data.append(kind(*(cast(item) for cast, item in
zip(casts, ast.literal_eval(line)))))
if __name__ == '__main__':
main()

Python - Getting Attributes From A File of Constants

I have a file of constant variables that I need to query and I am not sure how to go about it.
I have a database query which is returning user names and I need to find the matching user name in the file of constant variables.
The file looks like this:
SALES_MANAGER_01 = {"user_name": "BO01", "password": "password", "attend_password": "BO001",
"csm_password": "SM001", "employee_num": "BOSM001"}
There is just a bunch of users just like the one above.
My function looks like this:
#attr("user_test")
def test_get_user_for_login(self):
application_code = 'BO'
user_from_view = self.select_user_for_login(application_code=application_code)
users = [d['USER'] for d in user_from_view]
user_with_ent = choice(users)
user_wo_ent = user_with_ent[-4:]
password = ""
global_users = dir(gum)
for item in global_users:
if user_wo_ent not in item.__getattr__("user_name"):
user_with_ent = choice(users)
user_wo_ent = user_with_ent[-4:]
else:
password = item.__getattr__("password")
print(user_wo_ent, password)
global_users = dir(gum) is my file of constants. So I know I am doing something wrong since I am getting an attribute error AttributeError: 'str' object has no attribute '__getattr__', I am just not sure how to go about resolving it.

You should reverse your looping as you want to compare each item to your match condition. Also, you have a dictionary, so use it to do some heavy lifting.
You need to add some imports
import re
from ast import literal_eval
I've changed the dir(gum) bit to be this function.
def get_global_users(filename):
gusers = {} # create a global users dict
p_key = re.compile(ur'\b\w*\b') # regex to get first part, e.g.. SALES_MANAGER_01
p_value = re.compile(ur'\{.*\}') # regex to grab everything in {}
with (open(filename)) as f: # open the file and work through it
for line in f: # for each line
gum_key = p_key.match(line) # pull out the key
gum_value = p_value.search(line) # pull out the value
''' Here is the real action. update a dictionary
with the match of gum_key and with match of gum_value'''
gusers[gum_key.group()] = literal_eval(gum_value.group())
return(gusers) # return the dictionary
The bottom of your existing code is replaced with this.
global_users = get_global_users(gum) # assign return to global_users
for key, value in global_users.iteritems(): # walk through all key, value pairs
if value['user_name'] != user_wo_ent:
user_with_ent = choice(users)
user_wo_ent = user_with_ent[-4:]
else:
password = value['password']

So a very simple answer was get the dir of the constants file then parsing over it like so:
global_users = dir(gum)
for item in global_users:
o = gum.__dict__[item]
if type(o) is not dict:
continue
if gum.__dict__[item].get("user_name") == user_wo_ent:
print(user_wo_ent, o.get("password"))
else:
print("User was not in global_user_mappings")

I was able to find the answer by doing the following:
def get_user_for_login(application_code='BO'):
user_from_view = BaseServiceTest().select_user_for_login(application_code=application_code)
users = [d['USER'] for d in user_from_view]
user_with_ent = choice(users)
user_wo_ent = user_with_ent[4:]
global_users = dir(gum)
user_dict = {'user_name': '', 'password': ''}
for item in global_users:
o = gum.__dict__[item]
if type(o) is not dict:
continue
if user_wo_ent == o.get("user_name"):
user_dict['user_name'] = user_wo_ent
user_dict['password'] = o.get("password")
return user_dict

how to loop list to pass string to function name in python

I'm trying to find the most efficient way to create different function name myfunction_a ,.. b , c with slightly different code ( input file name 'app/data/mydata_a.csv' ) so here below is the a function I got
def myfunction_a(request):
os.getcwd() # Should get this Django project root (where manage.py is)
fn = os.path.abspath(os.path.join(os.getcwd(),'app/data/mydata_a.csv'))
# TODO: Move to helper module
response_data = {}
data_format = 'tsv'
if data_format == 'json':
with open(fn, 'rb') as tsvin:
tsvin = csv.reader(tsvin, delimiter='\t')
for row in tsvin:
print 'col1 = %s col2 = %s' % (row[0], row[1])
response_data[row[0]] = row[1]
result = HttpResponse(json.dumps(response_data), content_type = 'application/json')
else:
with open(fn, 'rb') as tsvin:
buff = tsvin.read()
result = HttpResponse(buff, content_type = 'text/tsv')
return result
I want to be able to loop through my list and create multiple function name:
mylist = ['a','b','c' ... 'z' ]
def myfunction_a(request):
... ( 'app/data/mydata_a.csv' )
return request
to get final result of :
def myfunction_a => taking 'app/data/mydata_a.csv'
def myfunction_b => taking 'app/data/mydata_b.csv'
def myfunction_c => taking 'app/data/mydata_c.csv'
right now I just copy and past and change it. is there a better to do this ? Any recommendation would be appreciated. Thanks.

you can add a variable to a string with
"app/data/mydata_%s.csv" % (character)
so
for character in mylist:
print "app/data/mydata_%s.csv" % (character)
should append everytime another charcter at the place of %s
So since you want for every function use another string to get another file you can
do something like this:
def myfunction(label, request):
return "app/data/mydata_%s.csv" % (label)
so you get the function label at the end of your documentpath. Since you described that you
only want to change the name so that it equals to the function label, you only need another parameter and not a new function name

If you must have a special function name, you could do this. Though why you'd need to I'm not sure.
import functools, sys
namespace = sys._getframe(0).f_globals
def myfunction(label, request):
print request
return "app/data/mydata_%s.csv" % (label)
my_labels = ['a','b','c']
for label in my_labels:
namespace['myfunction_%s'%label] = functools.partial(myfunction, label)
print myfunction_a('request1')
print myfunction_b('request2')
Output is this:
request1
app/data/mydata_a.csv
request2
app/data/mydata_b.csv
Or possibly a better implementation would be:
class MyClass(object):
def __init__(self, labels):
for label in labels:
setattr(self, label, functools.partial(self._myfunction, label))
def _myfunction(self, label, request):
print request
return "app/data/mydata_%s.csv" % (label)
myfunction = MyClass(['a','b','c'])
print myfunction.c('request3')
Output is this:
request3
app/data/mydata_c.csv