Develop Reference

Trouble using configparser to pass API credentials

I have a config.ini file that looks like this
[REDDIT]
client_id = 'myclientid23jd934g'
client_secret = 'myclientsecretjf30gj5g'
password = 'mypassword'
user_agent = 'myuseragent'
username = 'myusername'
When I try to use reddit's API praw like this:
import configparser
import praw
class redditImageScraper:
def __init__(self, sub, limit):
config = configparser.ConfigParser()
config.read('config.ini')
self.sub = sub
self.limit = limit
self.reddit = praw.Reddit(client_id=config.get('REDDIT','client_id'),
client_secret=config.get('REDDIT','client_secret'),
password=config.get('REDDIT','password'),
user_agent=config.get('REDDIT','user_agent'),
username=config.get('REDDIT','username'))
def get_content(self):
submissions = self.reddit.subreddit(self.sub).hot(limit=self.limit)
for submission in submissions:
print(submission.id)
def main():
scraper = redditImageScraper('aww', 25)
scraper.get_content()
if __name__ == '__main__':
main()
I get this traceback
Traceback (most recent call last):
File "config.py", line 30, in <module>
main()
File "config.py", line 27, in main
scraper.get_content()
File "config.py", line 22, in get_content
for submission in submissions:
File "C:\Users\Evan\Anaconda3\lib\site-packages\praw\models\listing\generator.py", line 61, in __next__
self._next_batch()
File "C:\Users\Evan\Anaconda3\lib\site-packages\praw\models\listing\generator.py", line 71, in _next_batch
self._listing = self._reddit.get(self.url, params=self.params)
File "C:\Users\Evan\Anaconda3\lib\site-packages\praw\reddit.py", line 454, in get
data = self.request("GET", path, params=params)
File "C:\Users\Evan\Anaconda3\lib\site-packages\praw\reddit.py", line 627, in request
method, path, data=data, files=files, params=params
File "C:\Users\Evan\Anaconda3\lib\site-packages\prawcore\sessions.py", line 185, in request
params=params, url=url)
File "C:\Users\Evan\Anaconda3\lib\site-packages\prawcore\sessions.py", line 116, in _request_with_retries
data, files, json, method, params, retries, url)
File "C:\Users\Evan\Anaconda3\lib\site-packages\prawcore\sessions.py", line 101, in _make_request
params=params)
File "C:\Users\Evan\Anaconda3\lib\site-packages\prawcore\rate_limit.py", line 35, in call
kwargs['headers'] = set_header_callback()
File "C:\Users\Evan\Anaconda3\lib\site-packages\prawcore\sessions.py", line 145, in _set_header_callback
self._authorizer.refresh()
File "C:\Users\Evan\Anaconda3\lib\site-packages\prawcore\auth.py", line 328, in refresh
password=self._password)
File "C:\Users\Evan\Anaconda3\lib\site-packages\prawcore\auth.py", line 138, in _request_token
response = self._authenticator._post(url, **data)
File "C:\Users\Evan\Anaconda3\lib\site-packages\prawcore\auth.py", line 31, in _post
raise ResponseException(response)
prawcore.exceptions.ResponseException: received 401 HTTP response
However when I manually insert the credentials, my code runs exactly as expected. Also, if I run the line
print(config.get('REDDIT', 'client_id'))
I get the output 'myclientid23jd934g' as expected.
Is there some reason that praw won't allow me to pass my credentials using configparser?

Double check what your inputs to praw.Reddit are:
kwargs = dict(client_id=config.get('REDDIT','client_id'),
client_secret=config.get('REDDIT','client_secret'),
password=config.get('REDDIT','password'),
user_agent=config.get('REDDIT','user_agent'),
username=config.get('REDDIT','username')))
print(kwargs)
praw.Reddit(**kwargs)

You're overcomplicating configuration here — PRAW will take care of this for you.
If you rename config.ini to praw.ini, you can replace your whole initialization with just
self.reddit = praw.Reddit('REDDIT')
This is because PRAW will look for a praw.ini file and parse it for you. If you want to give the section a more descriptive name, make sure to update it in the praw.ini as well as in the single parameter passed to Reddit (which specifies the section of the file to use).
See https://praw.readthedocs.io/en/latest/getting_started/configuration/prawini.html.
As this page notes, values like username and password should not have quotation marks around them. For example,
password=mypassword
is correct, but
password="mypassword"
is incorrect.

how do I call `generate_presigned_post` from `ibm_boto3`?

I create credentials (tried both write and manager) on the web interface and include {"HMAC":true}
I have used these credentials for more basic actions such as put_object and upload_file successfully.
However, I cannot get generate_presigned_post to work. It generates the following error:
ibm_botocore.exceptions.UnsupportedSignatureVersionError: Signature
version is not supported: oauth-presign-post
I run the following code,
import ibm_boto3
from ibm_botocore.client import Config
class COSPresignedURL():
def __init__(self, config):
cfg = Config(signature_version='oauth', s3={"payload_signing_enabled":True})
self.cos = ibm_boto3.client(
's3',
ibm_api_key_id=config['api_key'],
ibm_service_instance_id=config['instance_id'],
ibm_auth_endpoint=config['auth_endpoint'],
endpoint_url=config['url_endpoint'],
config=cfg)
def generate(self, bucket, key, Fields=None, Conditions=None, ExpiresIn=300):
return self.cos.generate_presigned_post(bucket, key, Fields, Conditions, ExpiresIn)
def main():
config = {
"api_key" : "VALUE OF apikey FROM CLOUD CREDENTIALS",
"instance_id" : "VALUE OF resource_instance_id FROM CLOUD CREDENTIALS",
"auth_endpoint" : "https://iam.cloud.ibm.com/identity/token",
"url_endpoint" : "https://s3.eu-de.cloud-object-storage.appdomain.cloud"
}
bucket = 'somebucket'
poster = COSPresignedURL(config)
uri = poster.generate(bucket, 'somekey')
print(f'{uri}')
if __name__ == '__main__':
main()
which generates the following error in full,
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ibm_botocore/signers.py", line 149, in sign
auth = self.get_auth_instance(**kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ibm_botocore/signers.py", line 222, in get_auth_instance
signature_version=signature_version)
ibm_botocore.exceptions.UnknownSignatureVersionError: Unknown Signature Version: oauth-presign-post.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "tmp.py", line 35, in <module>
main()
File "tmp.py", line 30, in main
uri = poster.generate(bucket, 'somekey')
File "tmp.py", line 16, in generate
return self.cos.generate_presigned_post(bucket, key, Fields, Conditions, ExpiresIn)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ibm_botocore/signers.py", line 714, in generate_presigned_post
expires_in=expires_in)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ibm_botocore/signers.py", line 526, in generate_presigned_post
'PutObject', request, region_name, 'presign-post')
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ibm_botocore/signers.py", line 153, in sign
signature_version=signature_version)
ibm_botocore.exceptions.UnsupportedSignatureVersionError: Signature version is not supported: oauth-presign-post

I have that working as following. When I have created the client and connected to the COS endpoint, my final call looks like this:
theURL=cos.generate_presigned_url('get_object',
Params = {'Bucket': buckets[0],
'Key': objects[0]
},
ExpiresIn = 100)
It seems that I use that function with different parameters. I constructed the cos object as shown here:
cos = ibm_boto3.client('s3',
config["apikey"],
endpoint_url='https://'+cos_host,
aws_access_key_id=config["cos_hmac_keys"["access_key_id"],
aws_secret_access_key=config["cos_hmac_keys"]["secret_access_key"])
As you can see, I passed the HMAC details.

OAuth2 InvalidScopeError in Backend Application flow

I am trying to implement the Backend Application flow to get an access token from the Datahug API (Following instructions in https://api.datahug.com/#gettingstarted).
from oauthlib.oauth2 import BackendApplicationClient
from requests_oauthlib import OAuth2Session
def get_access_token():
token_url = 'https://apps.datahug.com/identity/connect/token'
client_id = CLIENT_ID
client_secret = CLIENT_SECRET
scope = 'datahug_api'
client = BackendApplicationClient(client_id=client_id)
client.prepare_request_body(scope=[scope])
oauth = OAuth2Session(client=client)
token = oauth.fetch_token(token_url=token_url,
client_id=client_id,
client_secret=client_secret)
return token
if __name__ == '__main__':
token = get_access_token()
print(token)
When running this code I'm getting an InvalidScopeError, namely
user:dh user$ python so_test.py
Traceback (most recent call last):
File "so_test.py", line 21, in <module>
token = get_access_token()
File "so_test.py", line 17, in get_access_token
client_secret=client_secret)
File "/Users/user/anaconda3/lib/python3.5/site-packages/requests_oauthlib/oauth2_session.py", line 244, in fetch_token
self._client.parse_request_body_response(r.text, scope=self.scope)
File "/Users/user/anaconda3/lib/python3.5/site-packages/oauthlib/oauth2/rfc6749/clients/base.py", line 409, in parse_request_body_response
self.token = parse_token_response(body, scope=scope)
File "/Users/user/anaconda3/lib/python3.5/site-packages/oauthlib/oauth2/rfc6749/parameters.py", line 376, in parse_token_response
validate_token_parameters(params)
File "/Users/user/anaconda3/lib/python3.5/site-packages/oauthlib/oauth2/rfc6749/parameters.py", line 383, in validate_token_parameters
raise_from_error(params.get('error'), params)
File "/Users/user/anaconda3/lib/python3.5/site-packages/oauthlib/oauth2/rfc6749/errors.py", line 325, in raise_from_error
raise cls(**kwargs)
oauthlib.oauth2.rfc6749.errors.InvalidScopeError: (invalid_scope)
It seems that the problem is in the value scope='datahug_api', but this is the value suggested in the datahug api. Any suggestions on how to solve this much appreciated.

You are missing scope in the oauth.fetch_token().
So, your variable token should be:
token = oauth.fetch_token(token_url=token_url,scope=scope,client_id=client_id,
client_secret=client_secret)

I had success adding the scope to the BackendApplicationClient as can be seen from the developer docs.
client = BackendApplicationClient(client_id=client_id, scope=scope)
oauth = OAuth2Session(client=client)
token = oauth.fetch_token(token_url=url, client_id=client_id, client_secret=client_secret)

Instagram api python

I have been trying to use the Instagram API with python. After getting valid access_token, client_id, client_secret, I have tried to use the function user_recent_media.
access_token -
access_token, user_info = unauthenticated_api.exchange_code_for_access_token(code)
setting connection
api = client.InstagramAPI(access_token=access_token, client_secret=client_secret)
The connection is established successfully and then I wanted to get the user_recent media:
recent_media, next = api.user_recent_media(user_id = user_info['id'], count = 10)
Then I get the following error:
Traceback (most recent call last):
File "C:/instagram/main.py", line 57, in <module>
recent_media, next = api.user_recent_media(user_id = user_info['id'], count = 10)
File "C:\Users\user-pc\Anaconda\lib\site-packages\instagram\bind.py", line 197, in _call
return method.execute()
File "C:\Users\user-pc\Anaconda\lib\site-packages\instagram\bind.py", line 189, in execute
content, next = self._do_api_request(url, method, body, headers)
File "C:\Users\user-pc\Anaconda\lib\site-packages\instagram\bind.py", line 151, in _do_api_request
obj = self.root_class.object_from_dictionary(entry)
File "C:\Users\user-pc\Anaconda\lib\site-packages\instagram\models.py", line 99, in object_from_dictionary
for comment in entry['comments']['data']:
KeyError: 'data'
Someone knows what is the problem? I have checked everywhere and couldn't find the reason for this error.

OAuth1 with requests_oauthlib produces URL encoding error

I am using the requests_oauthlib module for OAuth authentication.
Unfortunately I cannot reproduce the OAuth1 workflow acoording to step one in this tutorial: http://requests-oauthlib.readthedocs.org/en/latest/oauth1_workflow.html
If I try to obtain a fetch response it throws me the followin error:
Traceback (most recent call last):
File "/home/neumannr/test.py", line 18, in <module>
fetch_response = oauth.fetch_request_token(request_token_url)
File "/usr/lib/python2.7/site-packages/requests_oauthlib/oauth1_session.py", line 195, in fetch_request_token
token = self._fetch_token(url)
File "/usr/lib/python2.7/site-packages/requests_oauthlib/oauth1_session.py", line 264, in _fetch_token
token = dict(urldecode(self.post(url).text))
File "/usr/lib/python2.7/site-packages/oauthlib/common.py", line 135, in urldecode
raise ValueError('Not a valid urlencoded string.')
ValueError: Not a valid urlencoded string.
My testing code looks as follows:
#! /usr/bin/env python
# Using OAuth1Session
from requests_oauthlib import OAuth1Session
# Using OAuth1 auth helper
import requests
from requests_oauthlib import OAuth1
client_key = 'a'
client_secret = 'b'
request_token_url = 'https://api.twitter.com/oauth/request_token'
# Using OAuth1Session
oauth = OAuth1Session(client_key, client_secret=client_secret)
fetch_response = oauth.fetch_request_token(request_token_url)
Since the URL is not being URL encoded within the tutorial either, I do not understand why this happens. If I try to URL encode the URL like this:
#! /usr/bin/env python
# Using OAuth1Session
import urllib
from requests_oauthlib import OAuth1Session
# Using OAuth1 auth helper
import requests
from requests_oauthlib import OAuth1
client_key = 'a'
client_secret = 'b'
request_token_url = urllib.quote('https://api.twitter.com/oauth/request_token')
# Using OAuth1Session
oauth = OAuth1Session(client_key, client_secret=client_secret)
fetch_response = oauth.fetch_request_token(request_token_url)
I get a missing schema error:
Traceback (most recent call last):
File "/home/neumannr/test.py", line 19, in <module>
fetch_response = oauth.fetch_request_token(request_token_url)
File "/usr/lib/python2.7/site-packages/requests_oauthlib/oauth1_session.py", line 195, in fetch_request_token
token = self._fetch_token(url)
File "/usr/lib/python2.7/site-packages/requests_oauthlib/oauth1_session.py", line 264, in _fetch_token
token = dict(urldecode(self.post(url).text))
File "/usr/lib/python2.7/site-packages/requests/sessions.py", line 377, in post
return self.request('POST', url, data=data, **kwargs)
File "/usr/lib/python2.7/site-packages/requests/sessions.py", line 324, in request
prep = req.prepare()
File "/usr/lib/python2.7/site-packages/requests/models.py", line 222, in prepare
p.prepare_url(self.url, self.params)
File "/usr/lib/python2.7/site-packages/requests/models.py", line 291, in prepare_url
raise MissingSchema("Invalid URL %r: No schema supplied" % url)
requests.exceptions.MissingSchema: Invalid URL u'https%3A//api.twitter.com/oauth/request_token': No schema supplied
How do I use oauthlib_requests anyway?
Thanks,
Richard

The error can be a result of a failed attempt to decode the response. For more details, please see OAuthlib issue 124.