I am pulling stock data from an REST API, however the API doesn't supply a list of stocks they offer so my list and their list is different. I running my requests through nested loops and I'm trying to figure out how to except KeyError just move on to the next company in the list.
for stock in stock_list:
#queryString = "symbols:"+stock+" AND publishedAt:[2021-03-20 TO 2021-04-02]"
queryString = "source.id:sec-api AND symbols:" + stock
payload = {
"type": "filterArticles",
"queryString": queryString,
"from": 0,
"size": 2000
}
print(queryString)
# Format your payload to JSON bytes
jsondata = json.dumps(payload)
jsondataasbytes = jsondata.encode('utf-8')
# Instantiate the request
req = urllib.request.Request(API_ENDPOINT)
# Set the correct HTTP header: Content-Type = application/json
req.add_header('Content-Type', 'application/json; charset=utf-8')
# Set the correct length of your request
req.add_header('Content-Length', len(jsondataasbytes))
# Send the request to the API
response = urllib.request.urlopen(req, jsondataasbytes)
# Read the response
res_body = response.read()
# Transform the response into JSON
assets = json.loads(res_body.decode("utf-8"))
##### parse JSON Array into variables for SQL ######
articles = assets["articles"][0:50]
#print(articles)
for article in articles:
title = (article['title'][0:50])
stock_id =(article['id'][0:50])
date = (article['publishedAt'][0:50])
sources = (article['source'])
name = (sources['name'][0:50])
id = (sources['id'][0:50])
#print(title)
details = (article['details'])
company_name = (details['name'][0:50])
type = (details['type'][0:50])
#print(type)
cik = (details['cik'][0:50])
symbol = (details['ticker'][0:50])
link_sec = (details['linkToHtmlAnnouncement'][0:50])
description = (details['description'][0:50])
try:
cursor.execute("""
INSERT INTO stock_sec (stock_id, title, date, description, link_sec, type, symbol, company_name, cik)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",(stock_id[0:50], title[0:50], date[0:50], description[0:50], link_sec[0:50], type[0:50], symbol[0:50], company_name[0:50], cik[0:50]))
except Exception as e:
print(e)
Related
I have to get information about product stocks from marketplace's API via POST request.
API requires to send products IDs (sku) in url
Example:
https://api.market.com/campaigns/{campaignId}/warehouse/{warehouseId}/stocks/actual.json?sku=sku1&sku=sku2&sku=sku3
So I guess, I have to pass a dict like {'sku': '1', 'sku': '2', 'sku': '3'}
But of course, It's impossible to create a dict with same keys.
I don't know how to solve this task.
I made a function, using urllib (urlencode) that works. But it create an url with only last element in params.
params = {"sku": "ps-22-1", "sku2": "ps-22-7-2", "sku3": "ps-22-7-3"}
def get_stocks(self, campaign_id, warehouse_id, sku):
""" Method for parse stocks
Parameters:
campaign_id (int): client_id in Store
warehouse_id (int): warehouse_id in Warehouse
sku (str): product sku in Product
Returns:
data (json): JSON with stocks data """
url = f"{self.url}campaigns/{campaign_id}/warehouses/{warehouse_id}/stocks/actual.json?"
req = requests.get(url + urllib.parse.urlencode(sku),
headers=self.headers)
if req.status_code == 200:
return True, '200', req.json()
return False, req.json()["error"]["message"]
I keep products IDs in my DB in such model:
class Product(models.Model):
name = models.CharField(max_length=150)
sku = models.CharField(max_length=10)
In your code, replace these lines
req = requests.get(url + urllib.parse.urlencode(sku),
headers=self.headers)
with
payload = {"sku": "ps-22-1", "sku2": "ps-22-7-2", "sku3": "ps-22-7-3"}
req = requests.get('url', params=payload, headers=self.headers)
Create a dictionary which contains all the list of products IDs (sku), then pass it through the request as data, like shown below:
def get_stocks(self, campaign_id, warehouse_id, sku):
""" Method for parse stocks
Parameters:
campaign_id (int): client_id in Store
warehouse_id (int): warehouse_id in Warehouse
sku (str): product sku in Product
Returns:
data (json): JSON with stocks data """
url = f"{self.url}campaigns/{campaign_id}/warehouses/{warehouse_id}/stocks/actual.json?"
data = {"sku_ids":["ps-22-1","ps-22-7-2","ps-22-7-3"]}
req = requests.get(url + urllib.parse.urlencode(sku),
headers=self.headers,
data=data)
if req.status_code == 200:
return True, '200', req.json()
return False, req.json()["error"]["message"]
And You have to make the changes in the API too, to filter multiple values.
In the below example, I am using SqlAlchemy ORM. You can change yours according to your ORM.
data = request.data()
session().query(Product).filter(Product.sku.in_(data['sku_ids'])).all()
I solved the problem. The difficulty was that the skus needed to be taken from the database and substituted dynamically.
I take each sku and its value from the database, form a list of dicts and pass it to the get_stocks method.
So, this works:
def get_stocks(self, campaign_id, warehouse_id, sku):
url = f"{self.url}campaigns/{campaign_id}/warehouses/{warehouse_id}/stocks/actual.json?"
sku_list = []
for e in sku:
parsed_sku = urllib.parse.urlencode(e)
sku_list.append(parsed_sku)
s = '&'.join(sku_list)
try:
req = requests.get(url + s,
headers=self.headers)
if req.status_code != 200:
logger.warning(f"Yandex API returned "
f"{req.status_code}")
return []
return req.json()
except (Timeout, HTTPError, Exception) as e:
logger.warning(f"{e}, {e.__class__}")
[enter image description here][1]I have a dataframe that included my own data I've requested from Spotify (https://www.spotify.com/de/account/privacy/) Now, I have .json data, that I've converted to .csv, that is my StreamingHistory ( incl msplayed, timestamp, artist, trackName). I'd like to add the track_id or uri to the dataframe. Is that possible to a match of the trackName and artistName with the Spotify API? Because the API is kind of complex for me now.
Now I'd like to add a new column with the uri/track_id to then get the audio features of the song. As I've said already I don't know if it's possible to get the uri in this way.
[1]: https://i.stack.imgur.com/S2UGh.png
CLIENT_ID = ''
CLIENT_SECRET = ''
client_credentials_manager = SpotifyClientCredentials(client_id=cid, client_secret=secret)
sp = spotipy.Spotify(client_credentials_manager=client_credentials_manager)
AUTH_URL ="https://accounts.spotify.com/api/token"
auth_response = requests.post(AUTH_URL, {
'grant_type': 'client_credentials',
'client_id': CLIENT_ID,
'client_secret': CLIENT_SECRET,
})
#Convert response to JSON
auth_response_data = auth_response.json()
#Save the access token
access_token = auth_response_data['access_token']
#Need to pass access token into header to send properly formed GET request to API server
headers = {
'Authorization': 'Bearer {token}'.format(token=access_token)
}
sp=spotipy.Spotify(auth_manager=SpotifyClientCredentials(client_id="CLIENT_ID",client_secret="CLIENT_SECRET"))
BASE_URL = 'https://api.spotify.com/v1/'
df.to_csv(StreamingHistoryEditedComplete.csv)
new_df['songArtistID'] = new_df['artistName'] + ":" + new_df['trackName']
Error 'Could not process parameter: str( 11429016), it must be of type list/tuple/ dict'
totalCollectedWeightDate = RestAPI['API']['totalCollectedWeightDate']
today1 = datetime.now()
today = today1.strftime("%Y-%m-%d")
year =datetime.strftime(datetime.now() - timedelta(365), '%Y-%m-%d')
txt6 = totalCollectedWeightDate
xx6 = txt6.replace("2021-02-17",year)
yy6=xx6.replace("2022-03-17",today)
# Generate Access Token
api_url =AccessTokenAPI
headers = {"Content-Type":"application/json"}
response = requests.post(api_url, data=json.dumps(Body),
headers=headers2,verify=False)
response.json()
#print(response.json()["access_token"])
# getSLAComplaintsData API Response
auth_token=response.json()["access_token"]
hed = {'Authorization': 'Bearer ' + auth_token}
totalCollectedWeightDateAPI1 =yy6
response = requests.get(totalCollectedWeightDateAPI1, headers=hed,verify=False)
r=response.content
print(r)
print(type(r))
q6 gives me the output as 11429016 which is a string but its showing an error
q6=str(json.loads(r)['SWM']['data']['dumpweight'])
print(q6)
print(type(q6))
for totalCollectedWeightDateAPI1 in r:
c.execute(""" UPDATE totalCollectedWeightDate22 SET dumpweight=%s """, (str(q6)))
# print(q6)
conn.commit()
this is webhook
. i recive json form api with this . i need extract json tags and send to mysql database . problem is it not sent all json tags . i think it need loop at the step i tag with this step at code . thanks .
import json
import urllib.parse
import urllib.request
import mysql.connector
urls = ('/.*', 'hooks')
app = web.application(urls, globals())
class hooks:
def POST(self):
data = web.data()
print()
print('DATA RECEIVED:')
print(data)
print()
cts = data.decode('utf-8') #decode bytes to string
r1 = cts.replace(cts[:9], '')
parsed = urllib.parse.unquote_plus(r1) # ready for post
print(parsed)
print(cts)
print(type(cts))
myurl = "https://webhook.site/c0e861b0-3cc1-42c2-a0c6-54ad980b01b0"
req = urllib.request.Request(myurl)
req.add_header('Content-Type', 'application/json; charset=utf-8')
jsondata = parsed
jsondataasbytes = jsondata.encode('utf-8') # convert to be bytes
req.add_header('Content-Length', len(jsondataasbytes))
print(jsondataasbytes)
response = urllib.request.urlopen(req, jsondataasbytes)
test_dict = json.loads(parsed)[0]
print(type(test_dict))
# Extracting specific keys from dictionary <<<<<<THIS STEP>>>>>>>>>
indic_label = test_dict['indicator_label']
status = test_dict['status']
creation_date = test_dict['creation_date']
laststatus = test_dict['last_status']
base = test_dict['base_currency']
quote_currency = test_dict['quote_currency']
indic = test_dict['indicator']
prices = test_dict['prices']
mydb = mysql.connector.connect(
host="*",
user="*",
password="*",
database="*"
)
cursor = mydb.cursor()
cursor.execute("""INSERT INTO allcoins
(base,quote_currency , indic,status,laststatus,creation_date,prices,indic_label)
VALUES(%s,%s,%s,%s,%s,%s,%s,%s)""",
(base, quote_currency, indic, status, laststatus, creation_date, prices, indic_label))
mydb.commit()
cursor.close()
mydb.close()
return 'OK'
if __name__ == '__main__':
app.run()
you can post test data to this hook by this curl
curl -d "messages=%5B%7B%22values%22%3A+%7B%22momentum%22%3A+%220.00%22%7D%2C+%22exchange%22%3A+%22binance%22%2C+%22market%22%3A+%22BNT%2FETH%22%2C+%22base_currency%22%3A+%22BNT%22%2C+%22quote_currency%22%3A+%22ETH%22%2C+%22indicator%22%3A+%22momentum%22%2C+%22indicator_number%22%3A+0%2C+%22analysis%22%3A+%7B%22config%22%3A+%7B%22enabled%22%3A+true%2C+%22alert_enabled%22%3A+true%2C+%22alert_frequency%22%3A+%22once%22%2C+%22signal%22%3A+%5B%22momentum%22%5D%2C+%22hot%22%3A+0%2C+%22cold%22%3A+0%2C+%22candle_period%22%3A+%224h%22%2C+%22period_count%22%3A+10%7D%2C+%22status%22%3A+%22hot%22%7D%2C+%22status%22%3A+%22hot%22%2C+%22last_status%22%3A+%22hot%22%2C+%22prices%22%3A+%22+Open%3A+0.000989+High%3A+0.000998+Low%3A+0.000980+Close%3A+0.000998%22%2C+%22lrsi%22%3A+%22%22%2C+%22creation_date%22%3A+%222020-05-10+16%3A16%3A23%22%2C+%22hot_cold_label%22%3A+%22%22%2C+%22indicator_label%22%3A+%22%22%2C+%22price_value%22%3A+%7B%22open%22%3A+0.000989%2C+%22high%22%3A+0.000998%2C+%22low%22%3A+0.00098%2C+%22close%22%3A+0.000998%7D%2C+%22decimal_format%22%3A+%22%25.6f%22%7D%2C+%7B%22values%22%3A+%7B%22leading_span_a%22%3A+%220.00%22%2C+%22leading_span_b%22%3A+%220.00%22%7D%2C+%22exchange%22%3A+%22binance%22%2C+%22market%22%3A+%22BNT%2FETH%22%2C+%22base_currency%22%3A+%22BNT%22%2C+%22quote_currency%22%3A+%22ETH%22%2C+%22indicator%22%3A+%22ichimoku%22%2C+%22indicator_number%22%3A+1%2C+%22analysis%22%3A+%7B%22config%22%3A+%7B%22enabled%22%3A+true%2C+%22alert_enabled%22%3A+true%2C+%22alert_frequency%22%3A+%22once%22%2C+%22signal%22%3A+%5B%22leading_span_a%22%2C+%22leading_span_b%22%5D%2C+%22hot%22%3A+true%2C+%22cold%22%3A+true%2C+%22candle_period%22%3A+%224h%22%2C+%22hot_label%22%3A+%22Bullish+Alert%22%2C+%22cold_label%22%3A+%22Bearish+Alert%22%2C+%22indicator_label%22%3A+%22ICHIMOKU+4+hr%22%2C+%22mute_cold%22%3A+false%7D%2C+%22status%22%3A+%22cold%22%7D%2C+%22status%22%3A+%22cold%22%2C+%22last_status%22%3A+%22cold%22%2C+%22prices%22%3A+%22+Open%3A+0.000989+High%3A+0.000998+Low%3A+0.000980+Close%3A+0.000998%22%2C+%22lrsi%22%3A+%22%22%2C+%22creation_date%22%3A+%222020-05-10+16%3A16%3A23%22%2C+%22hot_cold_label%22%3A+%22Bearish+Alert%22%2C+%22indicator_label%22%3A+%22ICHIMOKU+4+hr%22%2C+%22price_value%22%3A+%7B%22open%22%3A+0.000989%2C+%22high%22%3A+0.000998%2C+%22low%22%3A+0.00098%2C+%22close%22%3A+0.000998%7D%2C+%22decimal_format%22%3A+%22%25.6f%22%7D%2C+%7B%22values%22%3A+%7B%22bbp%22%3A+%220.96%22%2C+%22mfi%22%3A+%2298.05%22%7D%2C+%22exchange%22%3A+%22binance%22%2C+%22market%22%3A+%22BNT%2FETH%22%2C+%22base_currency%22%3A+%22BNT%22%2C+%22quote_currency%22%3A+%22ETH%22%2C+%22indicator%22%3A+%22bbp%22%2C+%22indicator_number%22%3A+1%2C+%22analysis%22%3A+%7B%22config%22%3A+%7B%22enabled%22%3A+true%2C+%22alert_enabled%22%3A+true%2C+%22alert_frequency%22%3A+%22once%22%2C+%22candle_period%22%3A+%224h%22%2C+%22period_count%22%3A+20%2C+%22hot%22%3A+0.09%2C+%22cold%22%3A+0.8%2C+%22std_dev%22%3A+2%2C+%22signal%22%3A+%5B%22bbp%22%2C+%22mfi%22%5D%2C+%22hot_label%22%3A+%22Lower+Band%22%2C+%22cold_label%22%3A+%22Upper+Band+BB%22%2C+%22indicator_label%22%3A+%22Bollinger+4+hr%22%2C+%22mute_cold%22%3A+false%7D%2C+%22status%22%3A+%22cold%22%7D%2C+%22status%22%3A+%22cold%22%2C+%22last_status%22%3A+%22cold%22%2C+%22prices%22%3A+%22+Open%3A+0.000989+High%3A+0.000998+Low%3A+0.000980+Close%3A+0.000998%22%2C+%22lrsi%22%3A+%22%22%2C+%22creation_date%22%3A+%222020-05-10+16%3A16%3A23%22%2C+%22hot_cold_label%22%3A+%22Upper+Band+BB%22%2C+%22indicator_label%22%3A+%22Bollinger+4+hr%22%2C+%22price_value%22%3A+%7B%22open%22%3A+0.000989%2C+%22high%22%3A+0.000998%2C+%22low%22%3A+0.00098%2C+%22close%22%3A+0.000998%7D%2C+%22decimal_format%22%3A+%22%25.6f%22%7D%5D" -X POST http://192.168.30.1
Perhaps define all variables upfront because you still need them to write to the db, then check if they are in the response and update:
indic_label = ''
status = ''
creation_date = ''
laststatus = ''
base = ''
quote_currency = ''
indic =''
prices = ''
if test_dict['indicator_label']:
indic_label = test_dict['indicator_label']
if test_dict['status']:
status = test_dict['status']
...
...
If it's an all or none situation, you can check for one variable then exit, otherwise check for each of them.
CONSUMER_KEY = "*******"
CONSUMER_SECRET = "***"
ACCESS_KEY = "**"
ACCESS_SECRET = ****"
url = 'https://api.twitter.com/1.1/account/verify_credentials.json'
auth = OAuth1(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_KEY, ACCESS_SECRET)
requests.get(url, auth=auth)
r = requests.get('https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=coderFord&count=2',auth=auth)
res=r.json()
def id_al():
for item in res:
b=item['id']
print(b)
req_ya(b)
def req_ya(b):
params = {"ids": b}
ponse = requests.get("https://api.twitter.com/labs/1/tweets/metrics/private", params=params,auth=auth)
ak=ponse.json()
for itema in ak['data']:
for items in res:
k=itema['tweet']
p=itema['video']
data = {"created_at":items['created_at'],"text":items['text'],"tweet_id": itema['tweet_id'], "like_count": k['like_count'],"retweet_count": k['retweet_count'],"quote_count": k['quote_count'],"reply_count": k['reply_count'],"impression_count": k['impression_count'],"view_count":p['view_count']}
data_dict = json.dumps(data, indent = 4, sort_keys = True)
print(json.loads(data_dict))
collection.insert_one(data)
print(id_al())
Error: KeyError: 'video'
If the tweet I send is text, it doesn't send me the video view. And it gives an error.How can I control this
And I can't check because the type part of the tweet doesn't exist.
Response sample like this
https://developer.twitter.com/en/docs/labs/tweet-metrics/quick-start/get-tweets-metrics-private
You can first check there is 'video' in itema.keys() and then perform your logic inside this condition.
Replace this part,
for itema in ak['data']:
for items in res:
k=itema['tweet']
p=itema['video'] # here you are getting error.
with
for itema in ak['data']:
for items in res:
k=itema['tweet']
if 'video' in itema.keys():
p = itema['video']