I have a Dictionary with Key String (2019-10-28 13:21) and Value of Object (DataPoint)
import requests
import json
import time
symbol = "AAPL"
intraday_url = "https://www.alphavantage.co/query?function=TIME_SERIES_INTRADAY&symbol="+symbol+"&interval=1min&outputsize=full&apikey="+api_key
sma_url = "https://www.alphavantage.co/query?function=SMA&symbol="+symbol+"&interval=1min&time_period=180&series_type=open&apikey="+api_key
ema_url = "https://www.alphavantage.co/query?function=EMA&symbol="+symbol+"&interval=1min&time_period=15&series_type=open&apikey="+api_key
vwap_url = "https://www.alphavantage.co/query?function=VWAP&symbol="+symbol+"&interval=1min&apikey="+api_key
macd_url = "https://www.alphavantage.co/query?function=MACD&symbol="+symbol+"&interval=1min&series_type=open&apikey="+api_key
rsi_url = "https://www.alphavantage.co/query?function=RSI&symbol="+symbol+"&interval=1min&time_period=100&series_type=open&apikey="+api_key
adx_url = "https://www.alphavantage.co/query?function=ADX&symbol="+symbol+"&interval=1min&time_period=100&apikey="+api_key
class DataPoint:
def __init__(self, time):
# 2019-10-31 15:49:00 (original)
# 2019-10-31 15:49 (formatted)
formatted_time = time[0:len(time)-3]
self.time = formatted_time
self.open = None
self.high = None
self.low = None
self.close = None
self.volume = None
self.sma = None
self.ema = None
self.vwap = None
self.macd = None
self.rsi = None
self.adx = None
def addIntraday(self,open,high,low,close,volume):
self.open = open
self.high = high
self.low = low
self.close = close
self.volume = volume
def addTechnical(self,technical,value):
if technical == "SMA":
self.sma = value
elif technical == "EMA":
self.ema = value
elif technical == "VWAP":
self.vwap = value
elif technical == "MACD":
self.macd = value
elif technical == "RSI":
self.rsi = value
elif technical == "ADX":
self.adx = value
def getIntraday(dictionary):
url = intraday_url
response = requests.get(url)
json = response.json()
intraday = json.get("Time Series (1min)")
keys = intraday.keys()
for key in keys:
ts = intraday.get(key)
dp = DataPoint(key)
open = ts.get("1. open")
high = ts.get("2. high")
low = ts.get("3. low")
close = ts.get("4. close")
volume = ts.get("5. volume")
dp.addIntraday(open,high,low,close,volume)
dictionary[dp.time] = dp
def getTechnicals(dictionary):
urls = [sma_url, ema_url, vwap_url, macd_url, rsi_url, adx_url]
technicals = ["SMA","EMA","VWAP","MACD","RSI","ADX"]
i = 0
while (i < len(urls)):
response = requests.get(urls[i])
json = response.json()
tech = json.get("Technical Analysis: " + technicals[i])
if (tech == None):
print("Empty response, retrying in 10 seconds...")
time.sleep(10)
else:
print("Getting Technical Indicator: " + technicals[i])
keys = tech.keys()
for key in keys:
t = tech.get(key)
v = t.get(technicals[i])
if (dictionary.get(key) != None):
dictionary.get(key).addTechnical(technicals[i], v)
i += 1
def writeDictionaryToFile(dictionary):
filename = "datapoints.json"
fp = open(filename, "a")
json_dictionary = json.dumps(dictionary)
fp.write(json_dictionary)
print("Wrote results to file: " + filename)
dictionary = {}
getIntraday(dictionary)
getTechnicals(dictionary)
writeDictionaryToFile(dictionary)
Here is the error:
Traceback (most recent call last):
File "/Users/Jason/Dev/Python/neural-network-example/alphavantage.py", line 124, in <module>
writeDictionaryToFile(dictionary)
File "/Users/Jason/Dev/Python/neural-network-example/alphavantage.py", line 113, in writeDictionaryToFile
json_dictionary = json.dumps(dictionary)
File "/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/__init__.py", line 231, in dumps
return _default_encoder.encode(obj)
File "/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/encoder.py", line 199, in encode
chunks = self.iterencode(o, _one_shot=True)
File "/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/encoder.py", line 257, in iterencode
return _iterencode(o, 0)
File "/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/encoder.py", line 179, in default
raise TypeError(f'Object of type {o.__class__.__name__} '
TypeError: Object of type DataPoint is not JSON serializable
From my understanding, I can use json.dumps() on a common python datatype, string, int, array dictionary, etc. But I cannot use it on custom objects that I've created. I've done research and from my research, I have figured out to use myCustomObject.dict to make the object serializable. How can I use this when I am trying to serialize the entire dictionary?
I'm new to Python, I just can't figure this out. Any help is greatly appreciated.
It's possible to achieve this using a custom JSON serializer, but that may be overkill for your task. A simpler solution is to give your class a couple of methods to convert to JSON and back, via dictionaries.
Your class has quite a lot of fields, so I'll give a simpler example for a toy class, which you should be able to adapt for your purpose:
import json
class Example:
def __init__(self, x, y):
self.x = x
self.y = y
def to_json(self):
return json.dumps({
'x': self.x,
'y': self.y
})
#classmethod
def from_json(cls, s):
d = json.loads(s)
return cls(d['x'], d['y'])
Usage:
>>> ex = Example(1, 2)
>>> s = ex.to_json()
>>> s
'{"y": 2, "x": 1}'
>>> ex2 = Example.from_json(s)
>>> ex2.x
1
>>> ex2.y
2
json is a very portable format, but it's also restricted. Only the following things can be serialized with Json:
dicts - {} (all keys must be strings)
lists - []
strings - "string"
integers - 0, 1, 2, ...
True
False
None
So you'll have to transform your object to some combination of these things, and have code to transform it back.
If you are planning on ONLY using python, you may be interested in pickle, which can serialize arbitrary python objects, as long as it can import the modules in which they were defined. Do note that unpacking pickles from unknown sources can lead to remote code executions.
Related
The complete code is as follows:
from Crypto.Protocol.KDF import scrypt
from Crypto.Cipher import AES
from Crypto.Random import get_random_bytes
class transmitter():
def __init__(self):
self.random_password = None
self.message_plain = True
self.key = None
self.salt = None
self.password_option()
self.text_option()
self.encrypt()
def password_option(self):
while ( self.random_password == None ):
random = input("\nDo you want to generate a random symmetric key? (y/n)\n\n>>> ").strip().lower()
if random == "y":
self.random_password = True
self.random()
elif random == "n":
self.random_password = False
self.random()
else:
pass
def text_option(self):
if self.message_plain:
question = input("\nHow will you enter your message?\n\n[1] file\n\n[2] directly in the program\n\n>>> ").strip()
if question == "1":
path = input("\nEnter the file path\n\n>>> ")
name = path.split("\\")[-1]
with open(name,mode = "r") as self.message_plain:
self.message_plain.read().encode("utf-8")
elif question == "2":
self.message_plain = input("\nEnter your message\n\n>>> ").strip()
self.message_plain = self.message_plain.encode("utf-8")
def random(self):
if self.random_password:
password = "password".encode("utf-8")
self.salt = get_random_bytes(16)
self.key = scrypt(password, self.salt, 16, N=2**14, r=8, p=1)
else:
password = input("\nEnter your password\n\n>>> ").strip()
self.salt = get_random_bytes(16)
self.key = scrypt(password.encode("utf-8"), self.salt, 16, N=2**14, r=8, p=1)
def encrypt(self):
cipher = AES.new(self.key,AES.MODE_GCM)
cipher.update(b"header")
cipher_text,tag_mac = cipher.encrypt_and_digest(self.message_plain)
transmitted_message = cipher_text,tag_mac,self.salt,cipher.nonce
Receptor(transmitted_message)
class receiver():
def __init__(self,received_message):
# nonce = aes_cipher.nonce
self.cipher_text,self.tag_mac,self.salt,self.nonce = received_message
self.decrypt(self.cipher_text,self.tag_mac,self.salt,self.nonce)
def decrypt(self,cipher_text,tag_mac,salt,nonce):
try:
password = input("\nEnter your password\n\n>>> ").strip()
decryption_key = scrypt(password.encode("utf-8"), salt, 16, N=2**14, r=8, p=1)
cipher = AES.new(decryption_key,AES.MODE_GCM,nonce)
cipher.update(b"header")
plain_text = cipher.decrypt_and_verify(cipher_text,tag_mac)
plain_text = plain_text.decode("utf-8")
print(f"\nText -> {plain_text}\n")
except ValueError:
print("\nAn error has occurred..\n")
if __name__ == '__main__':
init = transmitter()
And the error is as follows:
Traceback (most recent call last):
File ".\crypto.py", line 109, in <module>
init = Emisor()
File ".\crypto.py", line 16, in __init__
self.encrypt()
File ".\crypto.py", line 74, in encrypt
cipher_text,tag_mac = cipher.encrypt_and_digest(self.message_plain)
File "C:\Users\EQUIPO\AppData\Local\Programs\Python\Python37-32\lib\site-packages\Crypto\Cipher\_mode_gcm.py", line 547, in encryp
t_and_digest
return self.encrypt(plaintext, output=output), self.digest()
File "C:\Users\EQUIPO\AppData\Local\Programs\Python\Python37-32\lib\site-packages\Crypto\Cipher\_mode_gcm.py", line 374, in encryp
t
ciphertext = self._cipher.encrypt(plaintext, output=output)
File "C:\Users\EQUIPO\AppData\Local\Programs\Python\Python37-32\lib\site-packages\Crypto\Cipher\_mode_ctr.py", line 189, in encryp
t
ciphertext = create_string_buffer(len(plaintext))
TypeError: object of type '_io.TextIOWrapper' has no len()
I have tried everything but honestly, I don't know what else to do.
Does anyone know what could be wrong?
Basically, what I want to do is save the result of reading a certain file in a variable. So that I can encrypt it. Fortunately, the rest of the code is doing well, only that one block of code (the first one I put in the question) that presents the error.
You need to change this:
with open(name, mode = "r") as self.message_plain:
self.message_plain.read().encode("utf-8")
into this:
with open(name, mode="r") as input_file:
self.message_plain = input_file.read().encode("utf-8")
The 1st with block is functionally equivalent to this:
self.message_plain = open(name, mode = "r")
self.message_plain.read().encode("utf-8")
self.message_plain.close()
which does not make sense because it just makes self.message_plain into a file object and does not save the actual contents read().
The 2nd with block is functionally equivalent to this:
input_file = open(name, mode = "r")
self.message_plain = input_file.read().encode("utf-8")
input_file.close()
which makes more sense.
Basically, you were using the with statement incorrectly, because you changed the type of self.message_plain to <class '_io.TextIOWrapper'>. You can check this by printing type(self.message_plain) after/outside of the with statement. But the encrypt_and_digest method is expecting a bytes-like sequence:
>>> help(cipher.encrypt_and_digest)
Help on method encrypt_and_digest in module Crypto.Cipher._mode_gcm:
encrypt_and_digest(plaintext, output=None) method of Crypto.Cipher._mode_gcm.GcmMode instance
Perform encrypt() and digest() in one step.
:Parameters:
plaintext : bytes/bytearray/memoryview
The piece of data to encrypt.
...
I'd like to also suggest improvements to your coding style.
Put spaces after commas.
Instead of this:
self.cipher_text,self.tag_mac,self.salt,self.nonce = received_message
Write it like this:
self.cipher_text, self.tag_mac, self.salt, self.nonce = received_message
Class names should use the CapWords convention.
Instead of:
class transmitter
Change it to
class Transmitter
Initialize your variables to the same type you expect them to be.
Instead of this:
self.message_plain = True
Initialize it to an empty string ("") or (None) instead.
I know there are loads of answers to this question but I'm still not getting it...
Following is sa_reporting.py
class saReport():
def __init__(self, body, to_table, normalise=False, date_col=None):
global directory
self.body = body
self.to_table = to_table
self.normalise = normalise
self.date_col = date_col if date_col is not None else []
directory = os.path.join('/Users','python', self.to_table)
if not os.path.exists(directory):
os.mkdir(directory)
def download_files(self, ...):
...
def download_reports(self, ...):
...
def get_files(self):
...
def read_file(self, file):
....
def load_to_db(self, sort_by=None): # THIS IS WHAT I THINK IS CAUSING THE ERROR
sort_by = sort_by if sort_by is not None else [] # THIS IS WHAT I TRIED TO FIX IT
def normalise_data(self, data):
dim_data = []
for row in data:
if row not in dim_data:
dim_data.append(row)
return dim_data
def convert_dates(self, data):
if self.date_col:
for row in data:
for index in self.date_col:
if len(row[index]) > 10:
row[index] = row[index][:-5].replace('T',' ')
row[index] = datetime.datetime.strptime(row[index], "%Y-%m-%d %H:%M:%S")
else:
row[index] = datetime.datetime.strptime(row[index], "%Y-%m-%d").date()
return data
print(f'\nWriting data to {self.to_table} table...', end='')
files = self.get_files()
for file in files:
print('Processing ' + file.split("sa360/",1)[1] + '...', end='')
csv_file = self.read_file(file)
csv_headers = ', '.join(csv_file[0])
csv_data = csv_file[1:]
if self.normalise:
csv_data = self.normalise_data(csv_data)
csv_data = self.convert_dates(csv_data)
if sort_by:
csv_data = sorted(csv_data, key=itemgetter(sort_by))
#...some other code that inserts into a database...
Executing the following script (sa_main.py):
import sa_reporting
from sa_body import *
dim_campaign_test = sa_reporting.saReport(
body=dim_campaign_body,
to_table='dimsa360CampaignTest',
normalise=True,
date_col=[4,5]
)
dim_campaign_test_download = dim_campaign_test.download_reports()
dim_campaign_test_download.load_to_db(sort_by=0) # THIS IS WHERE THE ERROR OCCURS
Output and error message:
Downloading reports...
The report is still generating...restarting
The report is ready
Processing...
Downloading fragment 0 for report AAAnOdc9I_GnxAB0
Files successfully downloaded
Traceback (most recent call last):
File "sa_main.py", line 43, in <module>
dim_campaign_test_download.load_to_db(sort_by=0)
AttributeError: 'NoneType' object has no attribute 'load_to_db'
Why am I getting this error? And how can I fix it?
I just want to make None be the default argument and if a user specifies the sort_by parameter then None will be replaced with whatever the user specifies (which should be an integer index)
This code would seem to suggest that dim_campaign_test_download is being set to None. As in the below line, you set it to the result of dim_campaign_test.download_reports(), it is likely that no reports are being found.
dim_campaign_test_download = dim_campaign_test.download_reports()
You might want to instead do the following, as dim_campaign_test is the saReport Object on which you probably want to operate:
dim_campaign_test.load_to_db(sort_by=0)
Basically, i'm trying to write a script to bruteforce a protected zip file in python that tries every character combination (i.e. aa,ba,ca,da etc). But after a few tries it retrieves a strange error, and i'm not being able to find nowhere a solution for it.
Program:
import zipfile as z
class zipVictim:
def __init__(self,file):
self.found = False
self.password = ''
self.file = file
self.extracted_file_list = []
def bruteforceAttack(self,start_char: int,end_char: int,length: int,deep_loop=False,print_mode=False):
"""
Doc
"""
def _loop_chain(self,file,start_char,end_char,length):
lList = []
sPass = ''
iAttempt = 1
for iInc in range(length):
lList.append(start_char)
while lList[len(lList)-1] < end_char:
for iInc2 in range(start_char,end_char):
for iInc3 in range(len(lList)):
sPass = sPass + chr(lList[iInc3])
if iInc3 == 0:
lList[iInc3] = lList[iInc3] + 1
elif lList[iInc3 -1] > end_char:
lList[iInc3] = lList[iInc3] + 1
lList[iInc3 -1] = start_char
try:
if print_mode:
print("Attempt %s, password: %s" % (iAttempt,sPass),end='\r')
iAttempt = iAttempt + 1
oFile.extractall(pwd=sPass.encode())
self.extracted_file_list = oFile.namelist()
self.password = sPass
self.found = True
return self.found
except RuntimeError:
pass
sPass = ''
return self.found
oFile = self.file
if not deep_loop:
_loop_chain(self,oFile,start_char,end_char,length)
else:
for iInc in range(length):
_loop_chain(self,oFile,start_char,end_char,iInc+1)
if __name__ == '__main__':
file = z.ZipFile('data.zip')
s = zipVictim(file)
s.bruteforceAttack(64,125,2,print_mode=True)
Error Retrieved:
Traceback (most recent call last):
File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Python36_64\lib\zipfile.py", line 925, in _read1
data = self._decompressor.decompress(data, n)
zlib.error: Error -3 while decompressing data: invalid block type
Does anybody know what trigger this error and how to solve it?
In my test file:
class TestMakeSoup(TestCase):
fixtures = ['deals_test_data.json']
def test_string_is_valid(self):
s = Retailer.objects.get(pk=1)
with open('/home/danny/PycharmProjects/askarby/deals/tests/BestBuyTest.html', 'r') as myfile:
text = myfile.read().replace('\n', '')
self.assertTrue(s.make_soup(text))
In the file it's testing:
class retailer():
'''
Retail site, drawn from database queryset object
'''
def __init__(self,r_object):
'''
Initializes retailer variables
obj -> nonetype
Precondition: r_object.currency == 3
Precondition: r_object.name != None
'''
assert len(r_object.currency) == 3, "{} must be a three-letter string (eg 'USD').".format(r_object.currency)
assert r_object.name != None, "Name must exist."
assert r_object.deal_container_css != None, "Title css must exist."
assert r_object.title_css != None, "Title css must exist."
assert r_object.price_css != None, "Price css must exist."
self.name = r_object.name
self.base_url = r_object.base_url
self.currency = r_object.currency
#dict containing css lookup values for various fields
self.css = {}
self.css['container'] = self.extract_css(r_object.deal_container_css)
self.css['title'] = self.extract_css(r_object.title_css)
self.css['product_model'] = self.extract_css(r_object.product_model_css)
self.css['price'] = self.extract_css(r_object.price_css)
self.css['old_price'] = self.extract_css(r_object.old_price_css)
self.css['brand'] = self.extract_css(r_object.brand_css)
self.css['image'] = self.extract_css(r_object.image_css)
self.css['description'] = self.extract_css(r_object.description_css)
self.css['exclude'] = self.extract_css(r_object.exclude_css)
self.css['shipping'] = self.extract_css(r_object.shipping_css)
#dict containing associated clearance urls for retailer
self.clearance_urls = self.get_clearance_urls()
#dict to house final list of deals
self.deals = {}
def __str__(self):
return self.name
def make_soup(self, text):
assert isinstance(text,str), "text must be string."
soup = bs4.BeautifulSoup(text, "html.parser")
if soup:
return soup
return False
The Retailer call refers to the Retailer model in my deals app.
I get this error:
Error
Traceback (most recent call last):
File "/home/danny/PycharmProjects/askarby/deals/tests/test_deals.py", line 101, in test_string_is_valid
self.assertTrue(s.make_soup(text))
AttributeError: 'Retailer' object has no attribute 'make_soup'
Why isn't make_soup running as a method?
The retailer class takes an object retrieved from the database. I retrieved the object from the database but didn't create a class with it.
def test_makesoup(self):
z = Retailer.objects.get(pk=1)
s = dealscan.retailer(z)
with open('/home/danny/PycharmProjects/askarby/deals/tests/BestBuyTest.html', 'r') as myfile:
text = myfile.read().replace('\n', '')
self.assertTrue(s.make_soup(text))
solves it.
I'm very new to Python, my requirement is that i have CQ webpage and need to update the status of BugID based on particular fields.
Here is the sample code i'm trying.
import httplib2
import json
import getpass
import urllib
from string import Template
from xml.dom.minidom import parseString
class Credentials():
def assign_user (self):
self._user = 'user'
def assign_passwd (self):
self._passwd = 'pawrd'
user_cred = Credentials()
class RestLink:
def __init__(self, link, baseline_cr= 'ENGR00xxxx'):
self._link = Template(link)
self.cr = baseline_cr
def get_link(self):
return self._link.safe_substitute(recordId=self.cr,
loginid=user_cred.get_user(),
password=user_cred.get_passwd())
class CQBase:
SERVER = 'cq.am.domain.net'
RESPONSE_OK = 'OK'
def __init__(self, logger):
self._logger = logger
def send_request(self):
data = ''
try:
conn = httplib2.HTTPConnectionWithTimeout(self.SERVER)
conn.request("GET", link)
res = conn.getresponse()
data = res.read()
if res.reason != self.RESPONSE_OK:
raise ParseException('Cannot execute request!')
conn.close()
except:
conn.close()
raise
return data
class CQIssueReader(CQBase):
VIEW_CR_LINK = '/cqweb/restapi/TSR/ENGR/RECORD/${recordId}?format=JSON&recordType=CR&loginId=${loginid}&password=${password}&noframes=true'
def __init__(self, cr, logger):
CQBase.__init__(self, logger)
self._cr = cr
self._headline = ''
self._subtype = ''
self._branch = ''
self._is_resolved = 0
self._is_integrated = 0
self.parse_cr()
def parse_cr(self):
self._is_resolved = False
self._is_integrated = False
data = self.send_request(RestLink(self.VIEW_CR_LINK, self._cr).get_link())
parsedData = json.loads(data)
for field in parsedData['fields']:
if field['FieldName'] == 'Headline':
self._headline = field['CurrentValue']
if field['FieldName'] == 'Integrated':
self._logger.log_details('\tIntegrated = ' + field['CurrentValue'])
if field['CurrentValue'] == 'Y':
self._is_integrated = True
if field['FieldName'] == 'State':
self._logger.log_details('\tState = ' + field['CurrentValue'])
if (field['CurrentValue'] == 'Resolved') or (field['CurrentValue'] == 'Closed')\
or (field['CurrentValue'] == 'Verified'):
self._is_resolved = True
if field['FieldName'] == 'Subtype':
self._subtype = field['CurrentValue']
if field['FieldName'] == 'BranchName':
self._branch = field['CurrentValue']
self._logger.log_details('\tBranchName = ' + self._branch)
def get_headline(self):
return self._headline
def get_subtype(self):
return self._subtype
def get_branch_name(self):
return self._branch
test = CQIssueReader(CQBase)
test_data = CQIssueReader.parse_cr()
print (test_data)
i get following error with above code:
Traceback (most recent call last):
File "test.py", line 97, in <module>
test = CQIssueReader(CQBase)
TypeError: __init__() missing 1 required positional argument: 'logger'
Kindly guide me where i'm going wrong.
According to def __init__(self, cr, logger): your Class needs a parameter called logger to work. In test = CQIssueReader(CQBase), you've not passed in a logger.