In my test file:
class TestMakeSoup(TestCase):
fixtures = ['deals_test_data.json']
def test_string_is_valid(self):
s = Retailer.objects.get(pk=1)
with open('/home/danny/PycharmProjects/askarby/deals/tests/BestBuyTest.html', 'r') as myfile:
text = myfile.read().replace('\n', '')
self.assertTrue(s.make_soup(text))
In the file it's testing:
class retailer():
'''
Retail site, drawn from database queryset object
'''
def __init__(self,r_object):
'''
Initializes retailer variables
obj -> nonetype
Precondition: r_object.currency == 3
Precondition: r_object.name != None
'''
assert len(r_object.currency) == 3, "{} must be a three-letter string (eg 'USD').".format(r_object.currency)
assert r_object.name != None, "Name must exist."
assert r_object.deal_container_css != None, "Title css must exist."
assert r_object.title_css != None, "Title css must exist."
assert r_object.price_css != None, "Price css must exist."
self.name = r_object.name
self.base_url = r_object.base_url
self.currency = r_object.currency
#dict containing css lookup values for various fields
self.css = {}
self.css['container'] = self.extract_css(r_object.deal_container_css)
self.css['title'] = self.extract_css(r_object.title_css)
self.css['product_model'] = self.extract_css(r_object.product_model_css)
self.css['price'] = self.extract_css(r_object.price_css)
self.css['old_price'] = self.extract_css(r_object.old_price_css)
self.css['brand'] = self.extract_css(r_object.brand_css)
self.css['image'] = self.extract_css(r_object.image_css)
self.css['description'] = self.extract_css(r_object.description_css)
self.css['exclude'] = self.extract_css(r_object.exclude_css)
self.css['shipping'] = self.extract_css(r_object.shipping_css)
#dict containing associated clearance urls for retailer
self.clearance_urls = self.get_clearance_urls()
#dict to house final list of deals
self.deals = {}
def __str__(self):
return self.name
def make_soup(self, text):
assert isinstance(text,str), "text must be string."
soup = bs4.BeautifulSoup(text, "html.parser")
if soup:
return soup
return False
The Retailer call refers to the Retailer model in my deals app.
I get this error:
Error
Traceback (most recent call last):
File "/home/danny/PycharmProjects/askarby/deals/tests/test_deals.py", line 101, in test_string_is_valid
self.assertTrue(s.make_soup(text))
AttributeError: 'Retailer' object has no attribute 'make_soup'
Why isn't make_soup running as a method?
The retailer class takes an object retrieved from the database. I retrieved the object from the database but didn't create a class with it.
def test_makesoup(self):
z = Retailer.objects.get(pk=1)
s = dealscan.retailer(z)
with open('/home/danny/PycharmProjects/askarby/deals/tests/BestBuyTest.html', 'r') as myfile:
text = myfile.read().replace('\n', '')
self.assertTrue(s.make_soup(text))
solves it.
Related
I know there are loads of answers to this question but I'm still not getting it...
Following is sa_reporting.py
class saReport():
def __init__(self, body, to_table, normalise=False, date_col=None):
global directory
self.body = body
self.to_table = to_table
self.normalise = normalise
self.date_col = date_col if date_col is not None else []
directory = os.path.join('/Users','python', self.to_table)
if not os.path.exists(directory):
os.mkdir(directory)
def download_files(self, ...):
...
def download_reports(self, ...):
...
def get_files(self):
...
def read_file(self, file):
....
def load_to_db(self, sort_by=None): # THIS IS WHAT I THINK IS CAUSING THE ERROR
sort_by = sort_by if sort_by is not None else [] # THIS IS WHAT I TRIED TO FIX IT
def normalise_data(self, data):
dim_data = []
for row in data:
if row not in dim_data:
dim_data.append(row)
return dim_data
def convert_dates(self, data):
if self.date_col:
for row in data:
for index in self.date_col:
if len(row[index]) > 10:
row[index] = row[index][:-5].replace('T',' ')
row[index] = datetime.datetime.strptime(row[index], "%Y-%m-%d %H:%M:%S")
else:
row[index] = datetime.datetime.strptime(row[index], "%Y-%m-%d").date()
return data
print(f'\nWriting data to {self.to_table} table...', end='')
files = self.get_files()
for file in files:
print('Processing ' + file.split("sa360/",1)[1] + '...', end='')
csv_file = self.read_file(file)
csv_headers = ', '.join(csv_file[0])
csv_data = csv_file[1:]
if self.normalise:
csv_data = self.normalise_data(csv_data)
csv_data = self.convert_dates(csv_data)
if sort_by:
csv_data = sorted(csv_data, key=itemgetter(sort_by))
#...some other code that inserts into a database...
Executing the following script (sa_main.py):
import sa_reporting
from sa_body import *
dim_campaign_test = sa_reporting.saReport(
body=dim_campaign_body,
to_table='dimsa360CampaignTest',
normalise=True,
date_col=[4,5]
)
dim_campaign_test_download = dim_campaign_test.download_reports()
dim_campaign_test_download.load_to_db(sort_by=0) # THIS IS WHERE THE ERROR OCCURS
Output and error message:
Downloading reports...
The report is still generating...restarting
The report is ready
Processing...
Downloading fragment 0 for report AAAnOdc9I_GnxAB0
Files successfully downloaded
Traceback (most recent call last):
File "sa_main.py", line 43, in <module>
dim_campaign_test_download.load_to_db(sort_by=0)
AttributeError: 'NoneType' object has no attribute 'load_to_db'
Why am I getting this error? And how can I fix it?
I just want to make None be the default argument and if a user specifies the sort_by parameter then None will be replaced with whatever the user specifies (which should be an integer index)
This code would seem to suggest that dim_campaign_test_download is being set to None. As in the below line, you set it to the result of dim_campaign_test.download_reports(), it is likely that no reports are being found.
dim_campaign_test_download = dim_campaign_test.download_reports()
You might want to instead do the following, as dim_campaign_test is the saReport Object on which you probably want to operate:
dim_campaign_test.load_to_db(sort_by=0)
I have a Dictionary with Key String (2019-10-28 13:21) and Value of Object (DataPoint)
import requests
import json
import time
symbol = "AAPL"
intraday_url = "https://www.alphavantage.co/query?function=TIME_SERIES_INTRADAY&symbol="+symbol+"&interval=1min&outputsize=full&apikey="+api_key
sma_url = "https://www.alphavantage.co/query?function=SMA&symbol="+symbol+"&interval=1min&time_period=180&series_type=open&apikey="+api_key
ema_url = "https://www.alphavantage.co/query?function=EMA&symbol="+symbol+"&interval=1min&time_period=15&series_type=open&apikey="+api_key
vwap_url = "https://www.alphavantage.co/query?function=VWAP&symbol="+symbol+"&interval=1min&apikey="+api_key
macd_url = "https://www.alphavantage.co/query?function=MACD&symbol="+symbol+"&interval=1min&series_type=open&apikey="+api_key
rsi_url = "https://www.alphavantage.co/query?function=RSI&symbol="+symbol+"&interval=1min&time_period=100&series_type=open&apikey="+api_key
adx_url = "https://www.alphavantage.co/query?function=ADX&symbol="+symbol+"&interval=1min&time_period=100&apikey="+api_key
class DataPoint:
def __init__(self, time):
# 2019-10-31 15:49:00 (original)
# 2019-10-31 15:49 (formatted)
formatted_time = time[0:len(time)-3]
self.time = formatted_time
self.open = None
self.high = None
self.low = None
self.close = None
self.volume = None
self.sma = None
self.ema = None
self.vwap = None
self.macd = None
self.rsi = None
self.adx = None
def addIntraday(self,open,high,low,close,volume):
self.open = open
self.high = high
self.low = low
self.close = close
self.volume = volume
def addTechnical(self,technical,value):
if technical == "SMA":
self.sma = value
elif technical == "EMA":
self.ema = value
elif technical == "VWAP":
self.vwap = value
elif technical == "MACD":
self.macd = value
elif technical == "RSI":
self.rsi = value
elif technical == "ADX":
self.adx = value
def getIntraday(dictionary):
url = intraday_url
response = requests.get(url)
json = response.json()
intraday = json.get("Time Series (1min)")
keys = intraday.keys()
for key in keys:
ts = intraday.get(key)
dp = DataPoint(key)
open = ts.get("1. open")
high = ts.get("2. high")
low = ts.get("3. low")
close = ts.get("4. close")
volume = ts.get("5. volume")
dp.addIntraday(open,high,low,close,volume)
dictionary[dp.time] = dp
def getTechnicals(dictionary):
urls = [sma_url, ema_url, vwap_url, macd_url, rsi_url, adx_url]
technicals = ["SMA","EMA","VWAP","MACD","RSI","ADX"]
i = 0
while (i < len(urls)):
response = requests.get(urls[i])
json = response.json()
tech = json.get("Technical Analysis: " + technicals[i])
if (tech == None):
print("Empty response, retrying in 10 seconds...")
time.sleep(10)
else:
print("Getting Technical Indicator: " + technicals[i])
keys = tech.keys()
for key in keys:
t = tech.get(key)
v = t.get(technicals[i])
if (dictionary.get(key) != None):
dictionary.get(key).addTechnical(technicals[i], v)
i += 1
def writeDictionaryToFile(dictionary):
filename = "datapoints.json"
fp = open(filename, "a")
json_dictionary = json.dumps(dictionary)
fp.write(json_dictionary)
print("Wrote results to file: " + filename)
dictionary = {}
getIntraday(dictionary)
getTechnicals(dictionary)
writeDictionaryToFile(dictionary)
Here is the error:
Traceback (most recent call last):
File "/Users/Jason/Dev/Python/neural-network-example/alphavantage.py", line 124, in <module>
writeDictionaryToFile(dictionary)
File "/Users/Jason/Dev/Python/neural-network-example/alphavantage.py", line 113, in writeDictionaryToFile
json_dictionary = json.dumps(dictionary)
File "/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/__init__.py", line 231, in dumps
return _default_encoder.encode(obj)
File "/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/encoder.py", line 199, in encode
chunks = self.iterencode(o, _one_shot=True)
File "/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/encoder.py", line 257, in iterencode
return _iterencode(o, 0)
File "/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/encoder.py", line 179, in default
raise TypeError(f'Object of type {o.__class__.__name__} '
TypeError: Object of type DataPoint is not JSON serializable
From my understanding, I can use json.dumps() on a common python datatype, string, int, array dictionary, etc. But I cannot use it on custom objects that I've created. I've done research and from my research, I have figured out to use myCustomObject.dict to make the object serializable. How can I use this when I am trying to serialize the entire dictionary?
I'm new to Python, I just can't figure this out. Any help is greatly appreciated.
It's possible to achieve this using a custom JSON serializer, but that may be overkill for your task. A simpler solution is to give your class a couple of methods to convert to JSON and back, via dictionaries.
Your class has quite a lot of fields, so I'll give a simpler example for a toy class, which you should be able to adapt for your purpose:
import json
class Example:
def __init__(self, x, y):
self.x = x
self.y = y
def to_json(self):
return json.dumps({
'x': self.x,
'y': self.y
})
#classmethod
def from_json(cls, s):
d = json.loads(s)
return cls(d['x'], d['y'])
Usage:
>>> ex = Example(1, 2)
>>> s = ex.to_json()
>>> s
'{"y": 2, "x": 1}'
>>> ex2 = Example.from_json(s)
>>> ex2.x
1
>>> ex2.y
2
json is a very portable format, but it's also restricted. Only the following things can be serialized with Json:
dicts - {} (all keys must be strings)
lists - []
strings - "string"
integers - 0, 1, 2, ...
True
False
None
So you'll have to transform your object to some combination of these things, and have code to transform it back.
If you are planning on ONLY using python, you may be interested in pickle, which can serialize arbitrary python objects, as long as it can import the modules in which they were defined. Do note that unpacking pickles from unknown sources can lead to remote code executions.
I am doing a location search in Google App Engine and I want my search to be sorted based on proximity. I am getting the following error on the deployed version (production):
Search error:
Traceback (most recent call last):
File "/base/data/home/apps/s~sound-helper-87921/1.385231928987755902/application/search_handler.py", line 68, in doProductSearch
search_results = docs.Product.getIndex().search(search_query)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/datastore/datastore_rpc.py", line 105, in positional_wrapper
return wrapped(*args, **kwds)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/search/search.py", line 3676, in search
return self.search_async(query, deadline=deadline, **kwargs).get_result()
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/search/search.py", line 262, in get_result
return self._get_result_hook()
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/search/search.py", line 3690, in hook
_CheckStatus(response.status())
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/search/search.py", line 517, in _CheckStatus
raise _ERROR_MAP[status.code()](status.error_detail())
InvalidRequest: Failed to parse search request "distance(location, geopoint(30.008164999999998,-95.52959159999999)) < 2000"; Default text value is not appropriate for sort expression 'distance(location, geopoint(30.008165,-95.529592))'
The following is my code, which is pretty much copied from Google's tutorial:
def _buildQueryString(self, params):
userstr = string = params.get('querystr')
userprice = params.get('price')
userdist = params.get('less_than_distance')
loc = params.get('cur_location')
lat = loc.split(',')[0].split()[0]
lng = loc.split(',')[1].split()[0]
if userstr:
string = userstr
if userprice:
string = string + ' price < %s' % userprice
if userdist:
if not os.environ.get('SERVER_SOFTWARE','').startswith('Development'):
string = string + ' distance(%s, geopoint(%s,%s)) < %s' % (
docs.Product.LOCATION,lat,lng,userdist)
return string
def _buildQuery(self, params):
"""Build and return a search query object."""
user_query = self._buildQueryString(params)
doc_limit = self._getDocLimit()
try:
offsetval = int(params.get('offset', 0))
except ValueError:
offsetval = 0
loc = params.get('cur_location')
lat = loc.split(',')[0].split()[0]
lng = loc.split(',')[1].split()[0]
expr = 'distance(%s, geopoint(%f,%f))' % (docs.Product.LOCATION,float(lat),float(lng))
computed_expr_distance = search.FieldExpression(name='actual_distance',
expression=expr)
computed_expr_score = search.FieldExpression(name='actual_score',
expression='score')
returned_fields = [docs.Product.PID]
expr_list = []
expr_list.append(search.SortExpression(
expression=expr,
direction=search.SortExpression.ASCENDING,
default_value='2001'))
sortopts = search.SortOptions(expressions=expr_list, limit = doc_limit)
search_query = search.Query(
query_string=user_query.strip(),
options=search.QueryOptions(
limit=doc_limit,
offset=offsetval,
sort_options=sortopts,
returned_expressions=[computed_expr_distance],
returned_fields=returned_fields
)
)
return search_query
def doProductSearch(self, params):
"""Perform a product search and display the results."""
try:
search_query = self._buildQuery(params)
search_results = docs.Product.getIndex().search(search_query)
returned_count = len(search_results.results)
except search.Error:
logging.exception("Search error:")
msg = 'There was a search error (see logs).'
url = '/'
print('%s' % msg)
return [],[]
psearch_response = []
distances = []
# For each document returned from the search
for doc in search_results:
pdoc = docs.Product(doc)
for expr in doc.expressions:
if expr.name == 'actual_distance':
distances.append(expr.value)
pid = pdoc.getPID()
psearch_response.append(long(pid))
logging.debug('Distances: ' +str(distances))
return psearch_response, distances
Why is the Search API not recognizing my search query?
The problem was in my default_value. I modified the SortExpression to have an integer default_value instead of a string:
expr_list.append(search.SortExpression(
expression=expr,
direction=search.SortExpression.ASCENDING,
default_value=500000))
I'm very new to Python, my requirement is that i have CQ webpage and need to update the status of BugID based on particular fields.
Here is the sample code i'm trying.
import httplib2
import json
import getpass
import urllib
from string import Template
from xml.dom.minidom import parseString
class Credentials():
def assign_user (self):
self._user = 'user'
def assign_passwd (self):
self._passwd = 'pawrd'
user_cred = Credentials()
class RestLink:
def __init__(self, link, baseline_cr= 'ENGR00xxxx'):
self._link = Template(link)
self.cr = baseline_cr
def get_link(self):
return self._link.safe_substitute(recordId=self.cr,
loginid=user_cred.get_user(),
password=user_cred.get_passwd())
class CQBase:
SERVER = 'cq.am.domain.net'
RESPONSE_OK = 'OK'
def __init__(self, logger):
self._logger = logger
def send_request(self):
data = ''
try:
conn = httplib2.HTTPConnectionWithTimeout(self.SERVER)
conn.request("GET", link)
res = conn.getresponse()
data = res.read()
if res.reason != self.RESPONSE_OK:
raise ParseException('Cannot execute request!')
conn.close()
except:
conn.close()
raise
return data
class CQIssueReader(CQBase):
VIEW_CR_LINK = '/cqweb/restapi/TSR/ENGR/RECORD/${recordId}?format=JSON&recordType=CR&loginId=${loginid}&password=${password}&noframes=true'
def __init__(self, cr, logger):
CQBase.__init__(self, logger)
self._cr = cr
self._headline = ''
self._subtype = ''
self._branch = ''
self._is_resolved = 0
self._is_integrated = 0
self.parse_cr()
def parse_cr(self):
self._is_resolved = False
self._is_integrated = False
data = self.send_request(RestLink(self.VIEW_CR_LINK, self._cr).get_link())
parsedData = json.loads(data)
for field in parsedData['fields']:
if field['FieldName'] == 'Headline':
self._headline = field['CurrentValue']
if field['FieldName'] == 'Integrated':
self._logger.log_details('\tIntegrated = ' + field['CurrentValue'])
if field['CurrentValue'] == 'Y':
self._is_integrated = True
if field['FieldName'] == 'State':
self._logger.log_details('\tState = ' + field['CurrentValue'])
if (field['CurrentValue'] == 'Resolved') or (field['CurrentValue'] == 'Closed')\
or (field['CurrentValue'] == 'Verified'):
self._is_resolved = True
if field['FieldName'] == 'Subtype':
self._subtype = field['CurrentValue']
if field['FieldName'] == 'BranchName':
self._branch = field['CurrentValue']
self._logger.log_details('\tBranchName = ' + self._branch)
def get_headline(self):
return self._headline
def get_subtype(self):
return self._subtype
def get_branch_name(self):
return self._branch
test = CQIssueReader(CQBase)
test_data = CQIssueReader.parse_cr()
print (test_data)
i get following error with above code:
Traceback (most recent call last):
File "test.py", line 97, in <module>
test = CQIssueReader(CQBase)
TypeError: __init__() missing 1 required positional argument: 'logger'
Kindly guide me where i'm going wrong.
According to def __init__(self, cr, logger): your Class needs a parameter called logger to work. In test = CQIssueReader(CQBase), you've not passed in a logger.
I have a simple script using requests to validate a list of emails. Relevant code:
def ___process_email(email, output_file=None):
profile = request(email)
if profile and profile.success != 'nothing_useful':
logger.info('Found match for {0}'.format(email))
print(profile)
if output_file:
output_file.write(str(profile) + '\n')
else:
print("No information found\n")
This ran through 5 loops successfully then threw:
Traceback (most recent call last):
File "app.py", line 147, in <module> main()
File "app.py", line 141, in main ___process_email(arg, output)
File "app.py", line 107, in ___process_email if profile and profile.success != 'nothing_useful':
AttributeError: 'unicode' object has no attribute 'success'
Here's the model:
class Profile(object):
def __init__(self, person):
if person:
self.name = person.get('name')
self.jobinfo = [
(occupation.get('job_title'), occupation.get('company'))
for occupation in person.get('occupations', [])
]
self.memberships = [
(membership.get('site_name'), membership.get('profile_url'))
for membership in person.get('memberships', [])
]
self.success = person.get('success')
def __str__(self):
return dedent("""
Name: {0}
{1}
{2}
""").format(
self.name,
"\n".join(
"{0} {1}".format(title, company)
for title, company in self.jobinfo),
"\n".join(
"\t{0} {1}".format(site_name, url)
for site_name, url in self.memberships)
)
Request:
import requests
def request(email):
status_url = STATUS_URL.format(email)
response = requests.get(status_url).json()
session_token = response.get('session_token')
# fail gracefully if there is an error
if 'error' in response:
return response['error']
elif response['status'] == 200 and session_token:
logger.debug('Session token: {0}'.format(session_token))
url = URL.format(email)
headers = {'X-Session-Token': session_token}
response = requests.get(url, headers=headers).json()
if response.get('success') != 'nothing_useful':
return Profile(response.get('contact'))
return {}
Anyone see why my strings are unicode? thanks
If there is an error in the response, you return the error string:
if 'error' in response:
return response['error']
That's your unicode value there. Note that the same function returns either the 'error' value, a new Profile() instance, or an empty dictionary. You may want to make this more consistent, return only Profile() istances and None instead.
Instead of the error string, raise an exception and handle the exception in your ___process_email method:
class EmailValidationError(Exception):
pass
and in your request() function:
if 'error' in response:
raise EmailValidationError(response['error'])
then handle this in __process_email() with something like:
try:
profile = request(email)
if profile and profile.success != 'nothing_useful':
logger.info('Found match for {0}'.format(email))
print(profile)
if output_file:
output_file.write(str(profile) + '\n')
else:
print("No information found\n")
except EmailValidationError:
# Do something here