Develop Reference

Expand python function to return onee more value

I have this script to perform config-backups from juniper-devices based on input from Netbox. I would like to expand this script a little to fetch the firmware version from the juniper device and update our netbox using a custom field.
#!/usr/bin/python3
import sys,os,getopt
from getpass import getpass
from jnpr.junos import Device
import jnpr.junos.facts
from jnpr.junos.utils.config import Config
from jnpr.junos.exception import *
from lxml import etree
from pprint import pprint
import pynetbox
import datetime
nb = pynetbox.api(url='https://netbox-test/', token='<censored>')
save_path = '/config-backups/'
def printProgress(logtype,hostname,message):
print("%s:%s:%s"%(logtype,hostname,message))
def GetConfig(my_hostname, my_username, my_password):
try:
printProgress("INFO",my_hostname,"Connecting to device")
dev=Device(host=my_hostname,user=my_username,password=my_password)
dev.open(auto_probe=10)
dev.timeout=10
printProgress("INFO",my_hostname,"Retrieving config")
config = dev.rpc.get_config(options={'database':'committed','format':'set'})
junos_version = dev.facts['version']
configbackup = (etree.tostring(config, encoding='unicode', pretty_print=True))
completefilename = os.path.join(save_path, my_hostname+".set.config")
outfile=open(completefilename, "w")
outfile.write(configbackup)
dev.close()
outfile.close()
return True,junos_version
except Exception as err:
printProgress("ERROR",my_hostname,"Encountered exception while backing up config")
printProgress("ERROR",my_hostname,str(err))
return False
def main(argv):
junos_password = ''
try:
opts, args = getopt.getopt(argv,"?u:p:",["username=","password"])
except getopt.GetoptError:
print ('configbackup_junos.py -u <username> [-p <junos admin password>]')
sys.exit(2)
for opt, arg in opts:
if opt == '-?':
print ('configbackup_junos.py -u <username> [-p <junos admin password>]')
sys.exit()
elif opt in ("-u", "--username"):
junos_username = arg
elif opt in ("-p", "--password"):
junos_password = arg
print ('Will attempt to backup junos devices documented in Netbox using username:', junos_username)
if len(junos_password) > 0:
print ('Junos Password set on commandline\n')
else:
print ('password not entered, will ask for it')
junos_password=getpass(prompt="\nPassword: ")
nb_devicelist = nb.dcim.devices.all()
for nb_device in nb_devicelist:
platform = str(nb_device.platform)
pri_ip = str(nb_device.primary_ip)
asset = nb_device.asset_tag
devstatus = str(nb_device.status)
backup_enabled = nb_device.custom_fields['backup_enable']
if nb_device.virtual_chassis:
vchassismaster = str(nb_device.virtual_chassis.master)
else:
vchassismaster = 'no_chassis'
if backup_enabled == 'Yes' and platform == 'Junos' and devstatus == 'Active' and pri_ip != 'None' and asset:
if vchassismaster == (nb_device.name) or vchassismaster == 'no_chassis':
if GetConfig(asset,junos_username,junos_password):
print ("Config Successfully backed up from device.",nb_device)
nb_device.custom_fields['backup_status'] = "OK"
timenow = datetime.datetime.now()
timestamp = timenow.strftime("%Y-%m-d %X")
nb_device.custom_fields['backup_timestamp'] = timestamp
nb_device.save()
print (junos_version)
else:
printProgress ("ERROR",nb_device,"Config backup failed! ")
nb_device.custom_fields['backup_status'] = "FAILED"
nb_device.save()
print("")
if len(sys.argv) == 1:
sys.exit()
if __name__ == "__main__":
main(sys.argv[1:])
My problem is how do I get the variable "junos_version" returned from the function GetConfig. As you can see I have tried using "return True,junos_version", but how do I grab it in the output of the function?
(I have read all the articles I could find about this and tried a number of suggestions, but nothing works.
I need to be able to input the "junos_version" into this command
nb_device.custom_fields['firmware_version'] = junos_version
Which I wouldd placee just before the nb_device.save
I suspect it is my sense of logic that fails here, I just cannot see the forest for trees.

if GetConfig(asset,junos_username,junos_password):
you can change this with :
with flag, junos_version=GetConfig(asset,junos_username,junos_password):

If you return multiple values from a function, consider them a tuple.
In your case you can get the second value in the following way:
cfg = GetConfig(...) // add your args
junos_version = cfg[1] // get the 2-nd returned value from a tuple.

First you have to have a stable return.
def GetConfig(my_hostname, my_username, my_password):
try:
...
return True,junos_version
except Exception as err:
...
return False
In here you can have a tuple (of boolean and version) or a bool (True or False) as return.
I would change it as:
def GetConfig(my_hostname, my_username, my_password):
try:
...
return True,junos_version
except Exception as err:
...
return False, None
since there is no junos_version in exception.
Now you change the code where you use it as:
def main(argv):
...
if GetConfig(asset,junos_username,junos_password):
...
to
def main(argv):
...
cfg, version = GetConfig(asset,junos_username,junos_password)
if cfg:
...

Force an exception to be handled

I would like an exception to be thrown so I can complete coverage for a few lines.
def __query_items_from_db(self, my_id: str) -> list:
result = None
try:
result = self.table.query(
KeyConditionExpression='#id = :id',
ExpressionAttributeValues={
':id': my_id
},
ExpressionAttributeNames={
'#id': 'MY_ID'
}
)
except ClientError as e:
print('__query_items_from_db', e)
return result
This code works and won't throw an error as I have other code that sets up the table and and seeds data.
Here's what I tried to get the error to throw:
#mock_dynamodb2
def test_should_handle_an_error():
db_resource = create_mock_table()
module = CoverageReport(db_resource)
with pytest.raises(ClientError) as e:
raise ClientError() <-- i don't think this is right
actual_result = module._CoverageReport__query_items_from_db(
1) <-- this should return None because the ClientError is fired
assert actual_result == None
Any ideas?

Turns out I was thinking about this the wrong way. I forced an error by not creating the table before the test executes so I can't "query" a non-existent table. Now I can check that my result is None.
def test_should_handle_an_error():
db_resource = boto3.resource('dynamodb')
module = CoverageReport(db_resource)
actual_result = module._CoverageReport__query_items_from_db('testtesttest')
assert actual_result == None

cookie_str = match.group(1).AttributeError: 'NoneType' object has no attribute 'group'

I am working on Stock predicting project.I want to download historical data from yahoo finance and save them in CSV format.
Since I am beginner in Python I am unable to correct the error.
My code is as follows:
import re
import urllib2
import calendar
import datetime
import getopt
import sys
import time
crumble_link = 'https://finance.yahoo.com/quote/{0}/history?p={0}'
crumble_regex = r'CrumbStore":{"crumb":"(.*?)"}'
cookie_regex = r'Set-Cookie: (.*?); '
quote_link = 'https://query1.finance.yahoo.com/v7/finance/download/{}?period1={}&period2={}&interval=1d&events=history&crumb={}'
def get_crumble_and_cookie(symbol):
link = crumble_link.format(symbol)
response = urllib2.urlopen(link)
match = re.search(cookie_regex, str(response.info()))
cookie_str = match.group(1)
text = response.read()
match = re.search(crumble_regex, text)
crumble_str = match.group(1)
return crumble_str, cookie_str
def download_quote(symbol, date_from, date_to):
time_stamp_from = calendar.timegm(datetime.datetime.strptime(date_from, "%Y-%m-%d").timetuple())
time_stamp_to = calendar.timegm(datetime.datetime.strptime(date_to, "%Y-%m-%d").timetuple())
attempts = 0
while attempts < 5:
crumble_str, cookie_str = get_crumble_and_cookie(symbol)
link = quote_link.format(symbol, time_stamp_from, time_stamp_to, crumble_str)
#print link
r = urllib2.Request(link, headers={'Cookie': cookie_str})
try:
response = urllib2.urlopen(r)
text = response.read()
print "{} downloaded".format(symbol)
return text
except urllib2.URLError:
print "{} failed at attempt # {}".format(symbol, attempts)
attempts += 1
time.sleep(2*attempts)
return ""
if __name__ == '__main__':
print get_crumble_and_cookie('KO')
from_arg = "from"
to_arg = "to"
symbol_arg = "symbol"
output_arg = "o"
opt_list = (from_arg+"=", to_arg+"=", symbol_arg+"=")
try:
options, args = getopt.getopt(sys.argv[1:],output_arg+":",opt_list)
except getopt.GetoptError as err:
print err
for opt, value in options:
if opt[2:] == from_arg:
from_val = value
elif opt[2:] == to_arg:
to_val = value
elif opt[2:] == symbol_arg:
symbol_val = value
elif opt[1:] == output_arg:
output_val = value
print "downloading {}".format(symbol_val)
text = download_quote(symbol_val, from_val, to_val)
with open(output_val, 'wb') as f:
f.write(text)
print "{} written to {}".format(symbol_val, output_val)
And the Error message that I am getting is :
File "C:/Users/Murali/PycharmProjects/generate/venv/tcl/generate2.py", line
49, in <module>
print get_crumble_and_cookie('KO')
File "C:/Users/Murali/PycharmProjects/generate/venv/tcl/generate2.py", line
19, in get_crumble_and_cookie
cookie_str = match.group(1)
AttributeError: 'NoneType' object has no attribute 'group'
So how can we resolve this problem that has popped up?

Look at these two commands:
match = re.search(cookie_regex, str(response.info()))
cookie_str = match.group(1)
The first one takes the string response.info() does a regular expression search to match cookie_regex. Then match.group(1) is supposed to take the match from it. The problem however is that if you do a print match in between these commands, you'll see that the re.search() returned nothing. This means match.group() has nothing to "group", which is why it errors out.
If you take a closer look at response.info() (you could just add a print response.info() command in your script to see it), you'll see that there's a line in response code that starts with "set-cookie:", the code after which you're trying to capture. However, you have your cookie_regex string set to look for a line with "Set-Cookie:". Note the capital letters. When I change that string to all lower-case, the error goes away:
cookie_regex = r'set-cookie: (.*?); '
I did run into another error after that, where print "downloading {}".format(symbol_val) stops because symbol_val hasn't been defined. It seems that this variable is only declared and assigned when opt[2:] == symbol_arg:. So you may want to rewrite that part to cover all cases.

Base64 Decoding in Python 2.7

I am working with the following script that reads a csv file and decodes it. When I run it, I receive an Incorrect Padding error. When I take an individual entry and perform a str.decode('base64'), I get an appropriate result (no error). What could be the problem? Could some entries be corrupt and it's throwing the whole process off?
def get_sigpair_from_csv(csv_in, start=0, skip_to_tx=None, want_tx=[]):
want_tx=set(want_tx)
skip_entries = True
with open(csv_in,'r') as f:
for nr,line in enumerate(f):
if nr<start:
if nr%100000==0:
print "skip",nr,f.tell()
continue
if nr % 10000000 == 0:
print "10m", nr
try:
# read data
cols = line.split(";",1)
tx = cols[0].strip()
if skip_to_tx and tx==skip_to_tx:
skip_entries=False
# skip this entry - already in db
continue
if skip_to_tx and skip_entries:
print "skiptx",nr, tx
continue
if want_tx and tx not in want_tx:
continue
scriptsig = cols[1].decode("base64")
sig = scriptsig_to_ecdsa_sig(scriptsig)
sig['tx'] = tx
sig['nr'] = nr
yield sig
except ValueError, ve:
#print tx,repr(ve)
pass
except Exception, e:
print tx, repr(e)
Here is an example of the output:
879b068c75492e9d860763e843212d7aed2fb81ad3ee24592b48cdf5df624dcd Error('Incorrect padding',)
However, when I do:
x = '879b068c75492e9d860763e843212d7aed2fb81ad3ee24592b48cdf5df624dcd'
x.decode('base64')
It works

List indices must be integers error

I have the following error coming in my Python code:
if data['result'] == 0:
TypeError: list indices must be integers, not str
Following is the code :
data = urllib.urlencode(parameters)
req = urllib2.Request(url, data)
try:
response = urllib2.urlopen(req)
except urllib2.URLError, e:
self.redirect('/error')
json_post = response.read()
data = json.loads(json_post)
response_dict = simplejson.loads(json_post)
virustotal = VirusTotal()
if data['result'] == 0:
virustotal_result = True
elif data['result'] == -2:
self.response.out.write("API Request Rate Limit Exceeded<br/>")
elif data['result'] == -1:
self.response.out.write("API Key provided is wrong<br/>")
elif data['result'] == 1:
self.response.out.write("Time Stamp : %s<br/>"% data['report'][0])
I know that data is a list. So I changed it into integers as well but the code then showed the range error. Please Help??

When you say data = json.loads(json_post), it sounds like you got a list, not the dict you seem to be expecting.
If this isn't the problem, try updating with the full traceback and the value of json_post.

You are getting an array of objects back from response.read, which means you're getting a Python list. Try this:
data = json.loads(json_post)[0]