I created my first API where I can get data from my tables in Bigquery.
I can get all the data I need based on the 2 inputs below, but I am also trying to get the whole table when the inputs are empty, which I cannot do.
Thanks for your help
#app.route("/tracking", methods=['GET'])
def tracking_data():
haulier_id_tracking = request.args.get('haulier_id_tracking')
month_tracking = request.args.get('month_tracking')
query_job = bq_client.query("""
WITH t AS (
SELECT *
FROM mart.monthly_vehicle_stats
WHERE dt_fr_month = '{month_tracking}-01' AND (haulier_id_tracking = '{haulier_id_tracking}')
SELECT TO_JSON_STRING(STRUCT(ARRAY_AGG(STRUCT(dt_fr_month, haulier_id_tracking, vehicle_id , nb_days_tracked,
data_access, date_first_camp, invoiced)) AS data)) json
FROM t
""".format(month_tracking = month_tracking, haulier_id_tracking = haulier_id_tracking))
for row in query_job:
return json.loads(row["json"])
You can remove the where clause when the input is empty like this:
#app.route("/tracking", methods=['GET'])
def tracking_data():
haulier_id_tracking = request.args.get('haulier_id_tracking')
month_tracking = request.args.get('month_tracking')
where_clause = ''
if haulier_id_tracking != '' and month_tracking != '':
where_clause = f"WHERE dt_fr_month = '{month_tracking}-01' AND (haulier_id_tracking = '{haulier_id_tracking}'"
query_job = bq_client.query(f"""
WITH t AS (
SELECT * FROM mart.monthly_vehicle_stats {where_clause})
SELECT TO_JSON_STRING(STRUCT(ARRAY_AGG(STRUCT(dt_fr_month, haulier_id_tracking, vehicle_id , nb_days_tracked,
data_access, date_first_camp, invoiced)) AS data)) json
FROM t
""")
for row in query_job:
return json.loads(row["json"])
Related
Working on some code that uses pysimplegui as the UI and SQlite for the data sorting. I'm using SQLite's execute function to select data based on input from the user in the UI through variables. For example user wants to search for part name they input all or part of the name into the box, hit the search button which then runs my "parts_search" method, which will then only filter the result based on part name. OR the user enters information in multiple boxes which then filters based on the boxes that have information.
This here is runnable code provided you add a file base1.db in the same folder location as the script itself
import PySimpleGUI as sg
import os.path
import sqlite3
# sql var
c = None
conn = None
setup = None
# list var
parts = []
def sql():
global setup
conn_sql()
c.execute("""CREATE TABLE IF NOT EXISTS parts (part_name TEXT, part_number TEXT, part_series TEXT,
part_size INTEGER, job_type TEXT)""")
conn.commit()
if conn:
conn.close()
def conn_sql():
global c
global conn
# SQL connection var
if os.path.isfile('./base1.db'):
conn = sqlite3.connect('base1.db')
c = conn.cursor()
def main_gui_parts():
global parts
layout = [[sg.Text('Part Name: '), sg.Input(size=(20, 1), key='-PName-'), sg.Text('Part Series:'),
sg.Input(size=(10, 1), key='-PSeries-')],
[sg.Text('Part Number:'), sg.Input(size=(20, 1), key='-PNumber-'), sg.Text('Part Size:'),
sg.Input(size=(10, 1), key='-PSize-')],
[sg.Checkbox('Fit', key='-PFit-'), sg.Checkbox('Weld', key='-PWeld-'),
sg.Checkbox('Assemble', key='-PAssemble-'),
sg.Button('Search', key='-PSearch-')],
[sg.Listbox(parts, size=(58, 10), key='-PParts-')], [sg.Button('Back', key='-PBack-')]]
window = sg.Window('parts list', layout, grab_anywhere=True)
sql()
while True:
event, values = window.read()
if event == 'Close' or event == sg.WIN_CLOSED:
break
# PART WINDOW
part_name = values['-PName-']
part_series = values['-PSeries-']
part_number = values['-PNumber-']
part_size = values['-PSize-']
fit = values['-PFit-']
weld = values['-PWeld-']
assemble = values['-PAssemble-']
if event == '-PSearch-':
print('search parts')
part_search(part_name, part_series, part_number, part_size, fit, weld, assemble)
if event == '-PBack-':
break
window.close()
def part_search(part_name, part_series, part_number, part_size, fit, weld, assemble):
global parts
conn_sql()
filter_original = """SELECT * FROM parts WHERE """
filter = filter_original
if part_name:
print('part name: ' + part_name)
if filter == filter_original:
filter += """part_name LIKE ? """
else:
filter += """AND part_name LIKE ? """
if part_series:
print('part series: ' + part_series)
if filter == filter_original:
filter += """part_series=(?) """
else:
filter += """AND part_series=(?) """
if part_number:
print('part number: ' + part_number)
if filter == filter_original:
filter += """part_number LIKE ? """ ### DONT USE LIKE???
else:
filter += """AND part_number LIKE ? """ ### DONT USE LIKE???
if part_size:
print('part size: ' + part_size)
if filter == filter_original:
filter += """part_size=(?) """
else:
filter += """AND part_size=(?) """
if fit:
print('job type: ' + str(fit))
if filter == filter_original:
filter += """job_type = fit """
else:
filter += """AND job_type = fit """
if weld:
print('job type: ' + str(weld))
if filter == filter_original:
filter += """job_type = weld """
else:
filter += """AND job_type = weld """
if assemble:
print('job type: ' + str(assemble))
if filter == filter_original:
filter += """job_type = assemble"""
else:
filter += """AND job_type = assemble"""
print(filter)
#if filter != filter_original:
#c.execute(filter, ())
#else:
#c.execute("""SELECT * FROM parts""")
main_gui_parts()
THE PROBLEM: The commented code at the bottom is where I'm having trouble figuring out (in the "part_search" method). I don't use all of the variables all the time. Only filter with the variables provided by the user. which means the tuple should only have the variables which was input by the user.
If all the variables were used this is what it would look like. c.execute(filter, (part_name, part_series, part_number, part_size, fit, weld, assemble)) but more often than not only some of those variable will have been used and may need to look like this instead. c.execute(filter, (part_name, part_series, weld)) Somehow I need the variables here to be removeable(for lack of better word)
I've been learning a lot about SQLite but I could be seeing tunnel vision and can't think of a different way to go about this.
Probably the easiest way to deal with this is to put all the filter conditions and values into lists, and then only add a WHERE clause if the length of the filters list is non-zero. For example:
query = """SELECT * FROM parts"""
filters = []
values = []
if part_name:
filters.append("""part_name LIKE ?""")
values.append(part_name)
...
if len(filters):
query += ' WHERE ' + ' AND '.join(filters)
c.execute(query, tuple(values))
Note: should your filters ever include OR conditions, you need to parenthesise them when building the query to ensure correct operation i.e.
query += ' WHERE (' + ') AND ('.join(filters) + ')'
I'd like to do a and clause with two lists of multiple or clauses from the same table.
The problem with the following code is, that the query result is empty. If I just select 'indices' or 'brokers', the result is fine.
...
query = query.join(StockGroupTicker, on=(Ticker.id == StockGroupTicker.ticker))
# indices
if "indices" in filter:
where_indices = []
for f in filter["indices"]:
where_indices.append(StockGroupTicker.stock_index == int(f))
if len(where_indices):
query = query.where(peewee.reduce(peewee.operator.or_, where_indices))
# broker
if "brokers" in filter:
where_broker = []
for f in filter["brokers"]:
where_broker.append(StockGroupTicker.stock_index == int(f))
if len(where_broker):
query = query.where(peewee.reduce(peewee.operator.or_, where_broker))
return query.distinct()
SQL Querie (update)
# index and brocker
SELECT
DISTINCT `t1`.`id`,
`t1`.`symbol`,
`t1`.`type`,
`t1`.`name`,
`t1`.`sector`,
`t1`.`region`,
`t1`.`primary_exchange`,
`t1`.`currency`,
`t1`.`score`,
`t1`.`last_price`,
`t1`.`last_price_date`,
`t1`.`last_price_check`,
`t1`.`last_stock_split`,
`t1`.`next_earning`,
`t1`.`last_earnings_update`,
`t1`.`disused`,
`t1`.`source`,
`t1`.`source_intraday`,
`t1`.`created`,
`t1`.`modified`,
`t2`.`invest_score` AS `invest_score`
FROM
`ticker` AS `t1`
INNER JOIN `tickerstats` AS `t2` ON
(`t1`.`id` = `t2`.`ticker_id`)
INNER JOIN `stockgroupticker` AS `t3` ON
(`t1`.`id` = `t3`.`ticker_id`)
WHERE
(((((`t1`.`disused` IS NULL)
OR (`t1`.`disused` = 0))
AND (`t2`.`volume_mean_5` > 10000.0))
AND (`t3`.`stock_index_id` = 1))
AND (`t3`.`stock_index_id` = 10)
)
Thanks to #coleifer, the peewee solution is quite simple. I had to use an alias.
if "indices" in filter and filter["indices"]:
query = query.join(
StockGroupTicker, peewee.JOIN.INNER, on=(Ticker.id == StockGroupTicker.ticker)
)
where_indices = []
for f in filter["indices"]:
where_indices.append(StockGroupTicker.stock_index == int(f))
if len(where_indices):
query = query.where(peewee.reduce(peewee.operator.or_, where_indices))
if "brokers" in filter and filter["brokers"]:
BrokerGroupTicker = StockGroupTicker.alias()
query = query.join(
BrokerGroupTicker, peewee.JOIN.INNER, on=(Ticker.id == BrokerGroupTicker.ticker)
)
where_broker = []
for f in filter["brokers"]:
where_broker.append(BrokerGroupTicker.stock_index == int(f))
if len(where_broker):
query = query.where(peewee.reduce(peewee.operator.or_, where_broker))
return query.distinct()
Is there anyway i can compare two different databases (postgresl, sql server) and return the missing rows? I am missing one row in the postgresql table that is not in the sql server one and have no clue how to return that answer to me.
I have two connections opened for postgresql (bpo_table_results) and for sql server(rps_table_results)
postgresql table:
date count amount
1/1/21 500 1,234,654.12
sql server table:
date count amount
1/1/21 500 1,234,654.12
1/2/21 4541 3,457,787.24
expected results:
The row in the amount of 3,457,787.24 is missing from your posgresql table.
code:
def queryRPS(sql_server_conn, sql_server_cursor):
rps_item_count_l = []
rps_icl_amt_l = []
rps_table_q_2 = f"""select * from rps..sendfile where processingdate = '{cd}' and datasetname like '%ICL%' """
rps_table_results = sql_server_cursor.execute(rps_table_q_2).fetchall()
for row in rps_table_results:
rps_item_count = row[16]
rps_item_count_l.append(rps_item_count)
rps_icl_amt = row[18]
rps_icl_amt_l.append(rps_icl_amt)
def queryBPO(postgres_conn, postgres_cursor,rps_item_count_l, rps_icl_amt_l):
bpo_results_l = []
rps_results_l = []
for rps_count, rps_amount in zip(rps_item_count_l, rps_icl_amt_l):
rps_amount_f = str(rps_amount).rstrip('0')
rps_amount_f = ("{:,}".format(float(rps_amount_f)))
bpo_icl_awk_q_2 = """select * from ppc_data.icl_awk where num_items = '%s' and
file_total = '%s' """ % (str(rps_count), str(rps_amount_f))
postgres_cursor.execute(bpo_icl_awk_q_2)
bpo_table_results = postgres_cursor.fetchall()
rps_table_q_2 = f"""select * from rps..sendfile where processingdate = '{cd}' and datasetname like '%ICL%' """
rps_table_results = sql_server_cursor.execute(rps_table_q_2).fetchall()
rps_item_count_l, rps_icl_amt_l = queryRPS(sql_server_conn, sql_server_cursor)
queryBPO(postgres_conn, postgres_cursor, rps_item_count_l, rps_icl_amt_l)
I am new to python and I have a lot of variables I will be using in this script. These variables are being used to grab data from each column in an uploaded file. I have added variables for each object type and I have about 12 more object types to add. Isn't there a better way I can do this? I have the file it's grabbing data from here:
Action Object Solution ID hostgroup_name alias
Add Host Group ISD-CR ISD-CR_database ISD-CR Database
Add Service ISD-CR ISD-CR_database
Update Service Group ISD-CR ISD-CR Database
Delete Service ISD-CR ISD-CR_database
Here is the script I have so far.
from pynag import Model
from pynag.Parsers import config
def addObject():
# Add hostgroup object
hg = Model.Hostgroup()
hg.set_filename('/etc/nagios/objects/solution1/{0}.cfg'.format(target_hostgroup_name))
# Adding all attributes to allow any to be added if needed
hg.hostgroup_name = target_hostgroup_name
hg.alias = target_alias
hg.members = target_members
hg.hostgroup_members = target_hostgroup_members
hg.notes = target_notes
hg.notes_url = target_notes_url
hg.action_url = target_action_url
# Save
hg.save()
print "hostgroup added"
# Add service object
s = Model.Service()
s.set_filename('/etc/nagios/objects/solution1/{0}.cfg'.format(target_hostgroup_name))
# Adding all attributes to allow any to be added if needed
s.host_name = target_host_name
s.hostgroup_name = target_hostgroup_name
s.service_description = target_service_description
s.display_name = target_display_name
s.servicegroups = target_servicegroups
s.is_volatile = target_is_volatile
s.check_command = target_check_command
s.initial_state = target_initial_state
s.max_check_attempts = target_max_check_attempts
s.check_interval = target_check_interval
s.retry_interval = target_retry_interval
s.active_checks_enabled = target_active_checks_enabled
s.passive_checks_enabled = target_passive_checks_enabled
s.check_period = target_check_period
s.obsess_over_service = target_obsess_over_service
s.check_freshness = target_check_freshness
s.freshness_threshold = target_freshness_threshold
s.event_handler = target_event_handler
s.event_handler_enabled = target_event_handler_enabled
s.low_flap_threshold = target_low_flap_threshold
s.high_flap_threshold = target_high_flap_threshold
s.flap_detection_enabled = target_flap_detection_enabled
s.flap_detection_options = target_flap_detection_options
s.process_perf_data = target_process_perf_data
s.retain_status_information = target_retain_status_information
s.retain_nonstatus_information = target_retain_nonstatus_information
s.notification_interval = target_notification_interval
s.first_notification_delay = target_first_notification_delay
s.notification_period = target_notification_period
s.notification_options = target_notification_options
s.notification_enabled = target_notifications_enabled
s.contacts = target_contacts
s.contact_groups = target_contact_groups
s.stalking_options = target_stalking_options
s.notes = target_notes
s.notes_url = target_notes_url
s.action_url = target_action_url
s.icon_image = target_icon_image
s.icon_image_alt = target_icon_image_alt
# Save
s.save()
print "service added"
# Add servicegroup object
sg = Model.Servicegroup()
sg.set_filename('/etc/nagios/objects/solution1/{0}.cfg'.format(target_hostgroup_name))
# Adding all attributes to allow any to be added if needed
sg.servicegroup_name = target_servicegroup_name
sg.alias = target_alias
sg.members = target_members
sg.servicegroup_members = target_servicegroup_members
sg.notes = target_notes
sg.notes_url = target_notes_url
sg.action_url = '/etc/nagios/objects/solution1/{0}.cfg'.format(target_hostgroup_name)
# Save
sg.save()
print "service group added"
try:
current_file = csv.reader(open(input_file, "rb"), delimiter='\t')
except:
logging.error('No such file or directory. Please try again')
else:
for line in current_file:
for row in current_file:
target_hostgroup_name = row[3]
target_alias = row[4]
target_members = row[5]
target_hostgroup_members = row[6]
target_notes = row[7]
target_notes_url = row[8]
target_action_url = row[9]
target_host_name = row[10]
target_service_description = row[11]
target_display_name = row[12]
target_servicegroups = row[13]
target_is_volatile = row[14]
target_check_command = row[15]
target_initial_state = row[16]
target_max_check_attempts = row[17]
target_check_interval = row[18]
target_retry_interval = row[19]
target_active_checks_enabled = row[20]
target_passive_checks_enabled = row[21]
target_check_period = row[22]
target_obsess_over_service = row[23]
target_check_freshness = row[24]
target_freshness_threshold = row[25]
target_event_handler = row[26]
target_event_handler_enabled = row[27]
target_low_flap_threshold = row[28]
target_high_flap_threshold = row[29]
target_flap_detection_enabled = row[30]
target_flap_detection_options = row[31]
target_process_perf_data = row[32]
target_retain_status_information = row[33]
target_retain_nonstatus_information = row[34]
target_notification_interval = row[35]
target_first_notification_delay = row[36]
target_notification_period = row[37]
target_notification_options = row[38]
target_notifications_enabled = row[39]
target_contacts = row[40]
target_contact_groups = row[41]
target_stalking_options = row[42]
target_icon_image = row[43]
target_icon_image_alt = row[44]
target_servicegroup_name = row[45]
target_servicegroup_members = row[46]
If the values are in the same order every time, you could consider populating a list that you then could loop over, instead of doing it one by one.
For the "target" portion of your script, you could nest another loop for range(3, 46) as well, and pass the index to your list instead of manually for every number from 3 to 46.
Why do you do this?
for line in current_file:
for row in current_file:
If the first row is a header row and you're skipping it on purpose, you can use a DictReader instead.
It doesn't look like you'll be able to do much to clean this up, but you could factor out each "section" into its own function:
def save_hostgroup(name, alias, members, hostgroup_members, notes, notes_url, action_url):
hg = Model.Hostgroup()
hg.set_filename('/etc/nagios/objects/solution1/{0}.cfg'.format(target_hostgroup_name))
# Adding all attributes to allow any to be added if needed
hg.hostgroup_name = target_hostgroup_name
hg.alias = target_alias
hg.members = target_members
hg.hostgroup_members = target_hostgroup_members
hg.notes = target_notes
hg.notes_url = target_notes_url
hg.action_url = target_action_url
hg.save()
Behind the scenes all the member names of an object are stored in a dict. You can access this dict with vars(obj) or obj.__dict__. You can then use the update method of the dict to add a set of names to your object.
eg.
class SomeClass:
def __str__(self):
return "SomeClass({})".format(
", ".join(
"{}={!r}".format(key, value)
for key, value in self.__dict__.items()
)
)
__repr__ = __str__
target_names = ['var_a', 'var_b', 'var_c']
target_values = [1, 2, 3]
target = dict(zip(target_names, target_values))
assert target == {'var_a': 1, 'var_b': 2, 'var_c': 3}
s = SomeClass()
vars(s).update(target)
assert hasattr(s, 'var_a')
assert s.var_a == 1
print(s) # prints SomeClass(var_c=3, var_a=1, var_b=2)
I have a database table as follows. The data is in the form of a tree with
CREATE TABLE IF NOT EXISTS DOMAIN_HIERARCHY (
COMPONENT_ID INT NOT NULL ,
LEVEL INT NOT NULL ,
COMPONENT_NAME VARCHAR(127) NOT NULL ,
PARENT INT NOT NULL ,
PRIMARY KEY ( COMPONENT_ID )
);
The following data is in the table
(1,1,'A',0)
(2,2,'AA',1)
(3,2,'AB',1)
(4,3,'AAA',2)
(5,3,'AAB',2)
(6,3,'ABA',3)
(7,3,'ABB',3)
I have to retrieve the data and store in a python dictionary
I wrote the below code
conx = sqlite3.connect( 'nameofdatabase.db' )
curs = conx.cursor()
curs.execute( 'SELECT COMPONENT_ID, LEVEL, COMPONENT_NAME, PARENT FROM DOMAIN_HIERARCHY' )
rows = curs.fetchall()
cmap = {}
for row in rows:
cmap[row[0]] = row[2]
hrcy={}
for level in range( 1, maxl + 1 ):
for row in rows:
if row[1] == level:
if hrcy == {}:
hrcy[row[2]] = []
continue
parent = cmap[row[3]]
hrcy[parent].append( { row[2]: [] } )
The problem I'm facing is for nodes more than 2nd level ,they are getting added to the root instead of their parent ; where should I do the change in the code?
The problem is that you can't directly see the nodes for the second level after you insert them. Try this:
conx = sqlite3.connect( 'nameofdatabase.db' )
curs = conx.cursor()
curs.execute( 'SELECT COMPONENT_ID, LEVEL, COMPONENT_NAME, PARENT ' +
'FROM DOMAIN_HIERARCHY' )
rows = curs.fetchall()
cmap = {}
hrcy = None
for row in rows:
entry = (row[2], {})
cmap[row[0]] = entry
if row[1] == 1:
hrcy = {entry[0]: entry[1]}
# raise if hrcy is None
for row in rows:
item = cmap[row[0]]
parent = cmap.get(row[3], None)
if parent is not None:
parent[1][row[2]] = item[1]
print hrcy
By keeping each component's map of subcomponents in cmap, I can always reach each parent's map to add the next component to it. I tried it with the following test data:
rows = [(1,1,'A',0),
(2,2,'AA',1),
(3,2,'AB',1),
(4,3,'AAA',2),
(5,3,'AAB',2),
(6,3,'ABA',3),
(7,3,'ABB',3)]
The output was this:
{'A': {'AA': {'AAA': {}, 'AAB': {}}, 'AB': {'ABA': {}, 'ABB': {}}}}