Python: Unicode-objects must be encoded before hashing - python

I am working with an exchange for cryptocurrency which requires an encoding of the secret API key to gain access to private API calls. I have copied and pasted their Python code to begin executing my calls with it, but I receive this error every time I make a request.
TypeError: Unicode-objects must be encoded before hashing
I know what this means; I am a programmer. I can not find the root of the problem in the code I have received from the exchange, as I have not worked with hmac, hashlib, or base64. I have replaced all instances of the name of the exchange with the word "exchange" in the following code. No API keys are shown.
exchangeconfig = Exchange('key', 'secret')
base = 'https://exchange.com/'
def post_request(key, secret, path, data):
hmac_obj = hmac.new(secret, path + chr(0) + data, hashlib.sha512)
hmac_sign = base64.b64encode(hmac_obj.digest())
header = {
'Content-Type': 'application/json',
'User-Agent': 'exchangev2 based client',
'Rest-Key': key,
'Rest-Sign': hmac_sign,
}
proxy = ProxyHandler({'http': '127.0.0.1:8888'})
opener = build_opener(proxy)
install_opener(opener)
request = Request(base + path, data, header)
response = urlopen(request, data)
return json.load(response)
def gen_tonce():
return str(int(time.time() * 1e6))
class Exchange:
def __init__(self, key, secret):
self.key = key
self.secret = base64.b64decode(secret)
def request(self, path, params={}):
params = dict(params)
params['tonce'] = gen_tonce()
# data = urllib.urlencode(params)
data = json.dumps(params)
result = post_request(self.key, self.secret, path, data)
if result['result'] == 'success':
return result['data']
else:
raise Exception(result['result'])
exchangeconfig.request("api/3/account")
Please help me figure this out.
By the way: It seems to have a problem with this line in particular:
hmac_obj = hmac.new(secret, path + chr(0) + data, hashlib.sha512)
Thanks.
UPDATE: Fixed that error. Now onto this one:
TypeError: POST data should be bytes, an iterable of bytes, or a file object. It cannot be of type str.

These hash libraries deal with bytes objects, so you should encode your strings to bytes first (assuming the decoding end uses UTF-8):
hmac_obj = hmac.new(secret, path.encode('utf-8') + b'\0' + data.encode('utf-8'), hashlib.sha512)

Related

How to pass byte data in the xml payload

I am trying to pass a xml payload as an input to a function and in the payload objectZippedData field doesn't take string as an input, instead it needs the input to be in byte format.
def getSoapResponse(Envelope, url, action):
try:
envelope = Envelope
# Create and register opener. Requires proxy when behind a firewall
opener = urllib.request.build_opener(urllib.request.HTTPHandler(), urllib.request.HTTPSHandler(),
urllib.request.ProxyHandler())
urllib.request.install_opener(opener)
# Create request for the service call
request = urllib.request.Request(url)
# Configure the request content type to be xml
request.add_header("Content-Type", 'application/soap+xml;charset=utf-8')
# Set the SOAP action to be invoked; while the call works without this, the value is expected to be set based on standards
request.add_header("SOAPAction", action)
# Write the xml payload to the request
request.data = envelope.encode()
handle = urllib.request.urlopen(request, cafile=certifi.where())
# Get the response and process it
xmlString = (handle.read(2000).decode('utf-8'))
if action != "uploadObjectInSession":
# Convert the response into XML tree structure
sessionxml = fromstring(xmlString)
# Get the session value from the XML tree
ReturnVal = sessionxml[0][0][0].text if action == 'login' else 'Success'
return (ReturnVal)
else:
return "Success"
except Exception as err:
print("getSoapResponse - Error occurred; Message ", str(err))
print("===============================================================")
return "Error"
def uploadSession(catFile, Target):
try:
with open(catFile, 'rb') as f:
bytes_encoded = base64.b64encode(f.read())
string_encoded = bytes_encoded.decode()
f.close()
except IOError:
return "Error"
new_Envelope = """<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:v2="http://xmlns.oracle.com/oxp/service/v2">
<soapenv:Header/>
<soapenv:Body>
<v2:uploadObjectInSession>
<v2:reportObjectAbsolutePathURL>""" + Target + """</v2:reportObjectAbsolutePathURL>
<v2:objectType>xdoz</v2:objectType>
<v2:objectZippedData>""" + string_encoded + """</v2:objectZippedData>
<v2:bipSessionToken>""" + target_sessionid + """</v2:bipSessionToken>
</v2:uploadObjectInSession>
</soapenv:Body>
</soapenv:Envelope> """
Final = getSoapResponse(Envelope=new_Envelope, url, action='uploadObjectInSession')
if Final=="Error":
print("FAILURE")
return "Error"
else:
print(Final)
return "Success"
Once the above function is called, and if I pass the string in objectZippedData, it fails to succeed whereas if I pass byte data in objectZippedData, I get the below error
new_Envelope = """<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:v2="http://xmlns.oracle.com/oxp/service/v2">
TypeError: can only concatenate str (not "bytes") to str
Use .encode() to convert a string to bytes
>>> mystr="test"
>>> type(mystr)
<class 'str'>
>>> type(mystr.encode())
<class 'bytes'>
Your error suggests that when creating your newEnvelope by concatenating (using +) to accumulate your inputs together, one of them is actually of type bytes, instead of a string.
Find which one and use .decode() to turn it into a string.

Python (requests) - incorrect encoding when fetching headers

I am using requests library (python 3.9) to get filename from URL.[1] For some reason a file name is incorrectly encoded.
I should get "Ogłoszenie_0320.pdf" instead of "OgÅ\x82oszenie_0320.pdf".
My code looks something like this:
import requests
import re
def getFilenameFromRequest(url : str, headers):
# Parses from header information
contentDisposition = headers.get('content-disposition')
if contentDisposition:
filename = re.findall('filename=(.+)', contentDisposition)
print("oooooooooo: " + contentDisposition + " : " + str(filename))
if len(filename) != 0:
return filename[0]
# Parses from url
parsedUrl = urlparse(url)
return os.path.basename(parsedUrl.path)
def getFilenameFromUrl(url : str):
request = requests.head(url)
headers = request.headers
return getFilenameFromRequest(url, headers)
getFilenameFromUrl('https://przedszkolekw.bip.gov.pl'+
'/fobjects/download/880287/ogloszenie-uzp-nr-613234-pdf.html')
Any idea how to fix it?
I know for standard request I can set encoding directly:
request.encoding = 'utf-8'
But what am I supposed to do with this case?
[1]
https://przedszkolekw.bip.gov.pl/fobjects/download/880287/ogloszenie-uzp-nr-613234-pdf.html
Only characters from the ascii based latin-1 should be used as header values [rfc]. Here the file name has been escaped.
>>> s = "Ogłoszenie_0320.pdf"
>>> s.encode("utf8").decode("unicode-escape")
'OgÅ\x82oszenie_0320.pdf'
To reverse the process you can do
>>> sx = 'OgÅ\x82oszenie_0320.pdf'
>>> sx.encode("latin-1").decode("utf8")
'Ogłoszenie_0320.pdf'
(updated after conversation in comments)

Nested JSON Values cause "TypeError: Object of type 'int64' is not JSON serializable"

Would love some help here. Full context this is my first "purposeful" Python script. Prior to this I've only dabbled a bit and am honestly still learning so maybe I jumped in a bit too early here.
Long story short, been running all over fixing various type mismatches or just general indentation issues (dear lord python isn't forgiving on this).
I think I'm about finished but have a few last issues. Most of them seem to come from the same section too. This script is just mean to get a csv file that has 3 columns and use that to send requests based on the first column (either iOS or Android). The problem is when I'm creating the body to send...
Here's the code (a few tokens omitted for postability):
#!/usr/bin/python
# -*- coding: utf-8 -*-
import requests
import json
import pandas as pd
from tqdm import tqdm
from datetime import *
import uuid
import warnings
from math import isnan
import time
## throttling based on AF's 80 request per 2 minute rule
def throttle():
i = 0
while i <= 3:
print ("PAUSED FOR THROTTLING!" + "\n" + str(3-i) + " minutes remaining")
time.sleep(60)
i = i + 1
print (i)
return 0
## function for reformating the dates
def date():
d = datetime.utcnow() # # <-- get time in UTC
d = d.isoformat('T') + 'Z'
t = d.split('.')
t = t[0] + 'Z'
return str(t)
## function for dealing with Android requests
def android_request(madv_id,mtime,muuid,android_app,token,endpoint):
headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
params = {'api_token': token }
subject_identities = {
"identity_format": "raw",
"identity_type": "android_advertising_id",
"identity_value": madv_id
}
body = {
'subject_request_id': muuid,
'subject_request_type': 'erasure',
'submitted_time': mtime,
'subject_identities': dict(subject_identities),
'property_id': android_app
}
body = json.dumps(body)
res = requests.request('POST', endpoint, headers=headers,
data=body, params=params)
print("android " + res.text)
## function for dealing with iOS requests
def ios_request(midfa, mtime, muuid, ios_app, token, endpoint):
headers = {'Content-Type': 'application/json',
'Accept': 'application/json'}
params = {'api_token': token}
subject_identities = {
'identity_format': 'raw',
'identity_type': 'ios_advertising_id',
'identity_value': midfa,
}
body = {
'subject_request_id': muuid,
'subject_request_type': 'erasure',
'submitted_time': mtime,
'subject_identities': list(subject_identities),
'property_id': ios_app,
}
body = json.dumps(body)
res = requests.request('POST', endpoint, headers=headers, data=body, params=params)
print("ios " + res.text)
## main run function. Determines whether it is iOS or Android request and sends if not LAT-user
def run(output, mdf, is_test):
# # assigning variables to the columns I need from file
print ('Sending requests! Stand by...')
platform = mdf.platform
device = mdf.device_id
if is_test=="y":
ios = 'id000000000'
android = 'com.tacos.okay'
token = 'OMMITTED_FOR_STACKOVERFLOW_Q'
endpoint = 'https://hq1.appsflyer.com/gdpr/stub'
else:
ios = 'id000000000'
android = 'com.tacos.best'
token = 'OMMITTED_FOR_STACKOVERFLOW_Q'
endpoint = 'https://hq1.appsflyer.com/gdpr/opengdpr_requests'
for position in tqdm(range(len(device))):
if position % 80 == 0 and position != 0:
throttle()
else:
req_id = str(uuid.uuid4())
timestamp = str(date())
if platform[position] == 'android' and device[position] != '':
android_request(device[position], timestamp, req_id, android, token, endpoint)
mdf['subject_request_id'][position] = req_id
if platform[position] == 'ios' and device[position] != '':
ios_request(device[position], timestamp, req_id, ios, token, endpoint)
mdf['subject_request_id'][position] = req_id
if 'LAT' in platform[position]:
mdf['subject_request_id'][position] = 'null'
mdf['error status'][position] = 'Limit Ad Tracking Users Unsupported. Device ID Required'
mdf.to_csv(output, sep=',', index = False, header=True)
# mdf.close()
print ('\nDONE. Please see ' + output
+ ' for the subject_request_id and/or error messages\n')
## takes the CSV given by the user and makes a copy of it for us to use
def read(mname):
orig_csv = pd.read_csv(mname)
mdf = orig_csv.copy()
# Check that both dataframes are actually the same
# print(pd.DataFrame.equals(orig_csv, mdf))
return mdf
## just used to create the renamed file with _LOGS.csv
def rename(mname):
msuffix = '_LOG.csv'
i = mname.split('.')
i = i[0] + msuffix
return i
## adds relevant columns to the log file
def logs_csv(out, df):
mdf = df
mdf['subject_request_id'] = ''
mdf['error status'] = ''
mdf['device_id'].fillna('')
mdf.to_csv(out, sep=',', index=None, header=True)
return mdf
## solely for reading in the file name from the user. creates string out of filename
def readin_name():
mprefix = input('FILE NAME: ')
msuffix = '.csv'
mname = str(mprefix + msuffix)
print ('\n' + 'Reading in file: ' + mname)
return mname
def start():
print ('\nWelcome to GDPR STREAMLINE')
# # blue = OpenFile()
testing = input('Is this a test? (y/n) : ')
# return a CSV
name = readin_name()
import_csv = read(name)
output_name = rename(name)
output_file = logs_csv(output_name, import_csv)
run( output_name, output_file, testing)
# # print ("FILE PATH:" + blue)
## to disable all warnings in console logs
warnings.filterwarnings('ignore')
start()
And here's the error stacktrace:
Reading in file: test.csv
Sending requests! Stand by...
0%| | 0/384 [00:00<?, ?it/s]
Traceback (most recent call last):
File "a_GDPR_delete.py", line 199, in <module>
start()
File "a_GDPR_delete.py", line 191, in start
run( output_name, output_file, testing)
File "a_GDPR_delete.py", line 114, in run
android_request(device[position], timestamp, req_id, android, token, endpoint)
File "a_GDPR_delete.py", line 57, in android_request
body = json.dumps(body)
File "/Users/joseph/anaconda3/lib/python3.6/json/__init__.py", line 231, in dumps
return _default_encoder.encode(obj)
File "/Users/joseph/anaconda3/lib/python3.6/json/encoder.py", line 199, in encode
chunks = self.iterencode(o, _one_shot=True)
File "/Users/joseph/anaconda3/lib/python3.6/json/encoder.py", line 257, in iterencode
return _iterencode(o, 0)
File "/Users/joseph/anaconda3/lib/python3.6/json/encoder.py", line 180, in default
o.__class__.__name__)
TypeError: Object of type 'int64' is not JSON serializable
TL;DR:
Getting a typeError when calling this on a JSON with another nested JSON. I've confirmed that the nested JSON is the problem because if I remove the "subject_identities" section this compiles and works...but the API I'm using NEEDS those values so this doesn't actually do anything without that section.
Here's the relevant code again (and in the version I first used that WAS working previously):
def android (madv_id, mtime, muuid):
headers = {
"Content-Type": "application/json",
"Accept": "application/json"
}
params = {
"api_token": "OMMITTED_FOR_STACKOVERFLOW_Q"
}
body = {
"subject_request_id": muuid, #muuid,
"subject_request_type": "erasure",
"submitted_time": mtime,
"subject_identities": [
{ "identity_type": "android_advertising_id",
"identity_value": madv_id,
"identity_format": "raw" }
],
"property_id": "com.tacos.best"
}
body = json.dumps(body)
res = requests.request("POST",
"https://hq1.appsflyer.com/gdpr/opengdpr_requests",
headers=headers, data=body, params=params)
I get the feeling I'm close to this working. I had a much simpler version early on that worked but I rewrote this to be more dynamic and use less hard coded values (so that I can eventually use this to apply to any app I'm working with an not only the two it was made for).
Please be nice, I'm entirely new to python and also just rusty on coding in general (thus trying to do projects like this one)
You can check for numpy dtypes like so:
if hasattr(obj, 'dtype'):
obj = obj.item()
This will convert it to the closest equivalent data type
EDIT:
Apparently np.nan is JSON serializable so I've removed that catch from my answer
Thanks to everyone for helping so quickly here. Apparently I was deceived by the error message as the fix from #juanpa.arrivillaga did the job with one adjustment.
Corrected code was on these parts:
android_request(str(device[position]), timestamp, req_id, android, token, endpoint)
and here:
ios_request(str(device[position]), timestamp, req_id, ios, token, endpoint)
I had to cast to string apparently even though these values are not originally integers and tend to look like this instead ab12ab12-12ab-34cd-56ef-1234abcd5678

Why are my ZenDesk macros being updated, but no change actually going through?

I was trying to bulk edit the signature of my personal macros on ZenDesk, and the only way to do that is via the API. So I wrote this quick Python script to try to do it:
import sys
import time
import logging
import requests
import re
start_time = time.time()
# Set up logging
logger = logging.getLogger()
log_handler = logging.StreamHandler(sys.stdout)
log_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s - %(funcName)s - line %(lineno)d"))
log_handler.setLevel(logging.DEBUG)
logger.addHandler(log_handler)
logger.setLevel(logging.DEBUG)
def doTheGet(url, user, pwd):
response = requests.get(url, auth=(user + "/token", pwd))
if response.status_code != 200:
logger.error("Status: %s (%s) Problem with the request. Exiting. %f seconds elapsed" % (response.status_code, response.reason, time.time() - start_time))
exit()
data = response.json()
return data
def doThePut(url, updated_data, user, pwd):
response = requests.put(url, json="{'macro': {'actions': %r}}" % updated_data, headers={"Content-Type": "application/json"}, auth=(user + "/token", pwd))
if response.status_code != 200:
logger.error("Status: %s (%s) Problem with the request. Exiting. %f seconds elapsed" % (response.status_code, response.reason, time.time() - start_time))
exit()
data = response.json()
return data
def getMacros():
macros = {}
data = doTheGet("https://mydomain.zendesk.com/api/v2/macros.json", "me#mydomain.com", "111tokenZZZ")
def getMacros(macro_list, page, page_count):
if not page:
for macro in macro_list:
if macro["restriction"] and macro["active"]:
if macro["restriction"]["type"] == "User":
macros[macro["id"]] = macro["actions"]
else:
for macro in macro_list:
if macro["restriction"] and macro["active"]:
if macro["restriction"]["type"] == "User":
macros[macro["id"]] = macro["actions"]
page_count += 1
new_data = doTheGet(page, "me#mydomain.com", "111tokenZZZ")
new_macs = new_data["macros"]
new_next_page = new_data["next_page"]
getMacros(new_macs, new_next_page, page_count)
macs = data["macros"]
current_page = 1
next_page = data["next_page"]
getMacros(macs, next_page, current_page)
return macros
def updateMacros():
macros = getMacros()
regular = "RegEx to match signature to be replaced$" #since some macros already have the updated signature
for macro in macros:
for action in macros[macro]:
if action["field"] == "comment_value":
if re.search(regular, action["value"][1]):
ind = action["value"][1].rfind("\n")
action["value"][1] = action["value"][1][:ind] + "\nNew signature"
return macros
macs = updateMacros()
for mac in macs:
doThePut("https://mydomain.zendesk.com/api/v2/macros/%d.json" % (mac), macs[mac], "me#mydomain.com", "111tokenZZZ")
Now, everything's running as expected, and I get no errors. When I go to my macros on ZenDesk and sort them by last updated, I do see that the script did something, since they show as being last updated today. However, nothing changes on them. I made sure the data I'm sending over is edited (updateMacros is doing its job). I made sure the requests send back an OK response. So I'm sending updated data, getting back a 200 response, but the response sent back shows me the macros as they were before, with zero changes.
The only thing that occurs to me as maybe being wrong in some way is the format of the data I'm sending over, or something of the sort. But even then, I'd expect the response to not be a 200, then...
What am I missing here?
Looks like you're double-encoding the JSON data in your PUT request:
response = requests.put(url, json="{'macro': {'actions': %r}}" % updated_data, headers={"Content-Type": "application/json"}, auth=(user + "/token", pwd))
The json parameter expects an object, which it then dutifully encodes as JSON and sends as the body of the request; this is merely a convenience; the implementation is simply,
if not data and json is not None:
# urllib3 requires a bytes-like body. Python 2's json.dumps
# provides this natively, but Python 3 gives a Unicode string.
content_type = 'application/json'
body = complexjson.dumps(json)
if not isinstance(body, bytes):
body = body.encode('utf-8')
(source: https://github.com/kennethreitz/requests/blob/master/requests/models.py#L424)
Since the value is always passed through json.dumps(), if you pass a string representing already-encoded JSON it will itself be encoded:
"{\'macro\': {\'actions\': [{\'field\': \'comment_value\', \'value\': [\'channel:all\', \'Spiffy New Sig that will Never Be Saved\']}]}}"
ZenDesk, upon being given JSON it doesn't expect, updates the updated_at field and... Does nothing else. You can verify this by passing an empty string - same result.
Note that you're also relying on Python's repr formatting to fill in your JSON; that's probably a bad idea too. Instead, let's just reconstruct our macro object and let requests encode it:
response = requests.put(url, json={'macro': {'actions': updated_data}}, headers={"Content-Type": "application/json"}, auth=(user + "/token", pwd))
This should do what you expect.

Oauth1.0 API issue with Python

I'm trying to get magiccardmarket.eu API authentication to work in Python, but no matter whether I'm using rauth or requests_oauthlib, I get 403.
My code is:
#!/usr/bin/python
import logging
import rauth
import requests_oauthlib
logging.basicConfig(level=logging.DEBUG)
mkm_app_token = 'B7VI9Qg2xh855WtR'
mkm_app_secret = '<cut>'
mkm_access_token = 'LQj2rUwOFUJsmuJvCTlny1UzGZSXzHjo'
mkm_token_secret = '<cut>'
url = 'https://sandbox.mkmapi.eu/ws/v1.1/account'
# session = rauth.OAuth1Session(
# consumer_key=mkm_app_token,
# consumer_secret=mkm_app_secret,
# access_token=mkm_access_token,
# access_token_secret=mkm_token_secret,
# )
session = requests_oauthlib.OAuth1Session(
mkm_app_token,
client_secret=mkm_app_secret,
resource_owner_key=mkm_access_token,
resource_owner_secret=mkm_token_secret,
)
r = session.get(url)
print(r)
When I look at debugging info, everything seems fine (of course besides 403 response):
DEBUG:requests_oauthlib.oauth1_auth:Signing request <PreparedRequest [GET]> using client <Client nonce=None, signature_method=HMAC-SHA1, realm=None, encoding=utf-8, timestamp=None, resource_owner_secret=****, decoding=utf-8, verifier=None, signature_type=AUTH_HEADER, rsa_key=None, resource_owner_key=LQj2rUwOFUJsmuJvCTlny1UzGZSXzHjo, client_secret=****, callback_uri=None, client_key=B7VI9Qg2xh855WtR>
DEBUG:requests_oauthlib.oauth1_auth:Including body in call to sign: False
DEBUG:oauthlib.oauth1.rfc5849:Collected params: [(u'oauth_nonce', u'87129670621454425921416648590'), (u'oauth_timestamp', u'1416648590'), (u'oauth_consumer_key', u'B7VI9Qg2xh855WtR'), (u'oauth_signature_method', u'HMAC-SHA1'), (u'oauth_version', u'1.0'), (u'oauth_token', u'LQj2rUwOFUJsmuJvCTlny1UzGZSXzHjo')]
DEBUG:oauthlib.oauth1.rfc5849:Normalized params: oauth_consumer_key=B7VI9Qg2xh855WtR&oauth_nonce=87129670621454425921416648590&oauth_signature_method=HMAC-SHA1&oauth_timestamp=1416648590&oauth_token=LQj2rUwOFUJsmuJvCTlny1UzGZSXzHjo&oauth_version=1.0
DEBUG:oauthlib.oauth1.rfc5849:Normalized URI: https://sandbox.mkmapi.eu/ws/v1.1/account
DEBUG:oauthlib.oauth1.rfc5849:Base signing string: GET&https%3A%2F%2Fsandbox.mkmapi.eu%2Fws%2Fv1.1%2Faccount&oauth_consumer_key%3DB7VI9Qg2xh855WtR%26oauth_nonce%3D87129670621454425921416648590%26oauth_signature_method%3DHMAC-SHA1%26oauth_timestamp%3D1416648590%26oauth_token%3DLQj2rUwOFUJsmuJvCTlny1UzGZSXzHjo%26oauth_version%3D1.0
DEBUG:oauthlib.oauth1.rfc5849:Signature: 291LTesHZR6W4bjZ1NqSW5hEgoM=
DEBUG:oauthlib.oauth1.rfc5849:Encoding URI, headers and body to utf-8.
DEBUG:requests_oauthlib.oauth1_auth:Updated url: https://sandbox.mkmapi.eu/ws/v1.1/account
DEBUG:requests_oauthlib.oauth1_auth:Updated headers: {'Accept': '*/*', 'Connection': 'keep-alive', 'Accept-Encoding': 'gzip, deflate', 'Authorization': 'OAuth oauth_nonce="87129670621454425921416648590", oauth_timestamp="1416648590", oauth_version="1.0", oauth_signature_method="HMAC-SHA1", oauth_consumer_key="B7VI9Qg2xh855WtR", oauth_token="LQj2rUwOFUJsmuJvCTlny1UzGZSXzHjo", oauth_signature="291LTesHZR6W4bjZ1NqSW5hEgoM%3D"', 'User-Agent': 'python-requests/2.4.3 CPython/2.7.8 Darwin/14.0.0'}
DEBUG:requests_oauthlib.oauth1_auth:Updated body: None
INFO:requests.packages.urllib3.connectionpool:Starting new HTTPS connection (1): sandbox.mkmapi.eu
DEBUG:requests.packages.urllib3.connectionpool:"GET /ws/v1.1/account HTTP/1.1" 403 None
This is not an issue of authentication details, which are provided on account profile page when you request dedicated application API access, since those details work fine with PHP example provided by the site: https://www.mkmapi.eu/ws/documentation/API:Auth_libcurl
When I go through site's documentation, nothing seems out of ordinary: https://www.mkmapi.eu/ws/documentation/API:Auth_Overview
I honestly don't know where to go from here...
I realized that the code above with requests_oauthlib didn't build the header like it was layed out in the documentation, so I ended up inventing the wheel again and building the header myself, following the steps outlined in the documentation: https://www.mkmapi.eu/ws/documentation/API:Auth_OAuthHeader
The following script is not very beautiful, but it does its job.
import requests
from urllib import quote_plus as rawurlencode
import time
import string
import random
import operator
from hashlib import sha1
from hmac import new as hmac
def id_generator(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
# personal Info - taken from https://www.mkmapi.eu/ws/documentation/API:Auth_Overview
mkmAppToken = 'bfaD9xOU0SXBhtBP'
mkmAppSecret = 'pChvrpp6AEOEwxBIIUBOvWcRG3X9xL4Y'
mkmAccessToken = 'lBY1xptUJ7ZJSK01x4fNwzw8kAe5b10Q'
mkmAccessSecret = 'hc1wJAOX02pGGJK2uAv1ZOiwS7I9Tpoe'
# Url to access on mkm
# note that this deviates from the example in the header documentation (https://www.mkmapi.eu/ws/documentation/API:Auth_OAuthHeader) which uses
#accessUrl = 'https://www.mkmapi.eu/ws/v1.1/account'
accessUrl = 'https://www.mkmapi.eu/ws/v1.1/output.json/account'
#Method for access
MyMethod = "GET"
baseString = MyMethod + "&" + rawurlencode(accessUrl) + "&"
# create a random string
# the documentation in https://www.mkmapi.eu/ws/documentation/API:Auth_OAuthHeader uses
#nonce = 53eb1f44909d6
nonce = id_generator(8)
# what time is it?
# the documentation in https://www.mkmapi.eu/ws/documentation/API:Auth_OAuthHeader uses
#now = 1407917892
now = str(int(time.time()))
MyOauthmethod = "HMAC-SHA1"
MyOauthver = "1.0"
# define Parameters and values, order doesn't matter
paramDict ={"oauth_consumer_key":mkmAppToken, "oauth_token" :mkmAccessToken, "oauth_nonce":nonce, "oauth_timestamp":now, "oauth_signature_method":MyOauthmethod, "oauth_version":MyOauthver}
# sorting of parameters is done here
sorted_paramDict = sorted(paramDict.items(), key=operator.itemgetter(0))
#collect the full parameters string
paramStr = ''
for kv in sorted_paramDict:
paramStr = paramStr + kv[0] + "=" + kv[1] + "&"
# and get rid of the trailing ampersand
paramStr = paramStr[:-1]
#concatenate request and oauth parameters
baseString = baseString + rawurlencode(paramStr)
# concatenate both keys
signingKey = rawurlencode(mkmAppSecret) + "&" + rawurlencode(mkmAccessSecret)
# and create a hased signature with the key and the baseString
Signature = hmac(signingKey, baseString, sha1).digest().encode('base64')[:-1]
# construct the header from the parameters and the URL and the signature
MyHeader = 'OAuth ' + 'realm="' + accessUrl + '", '
for kv in sorted_paramDict:
MyHeader += kv[0] + '="' + kv[1] + '",'
MyHeader += 'oauth_signature="' + Signature +'"'
headers = {'Authorization': MyHeader}
# and now requests can do its magic (pun intended)
r = requests.get(accessUrl, headers=headers)
outjson = r.json()
You need to provide the realm as an argument to the OAuth1Session, like so:
session = requests_oauthlib.OAuth1Session(
mkm_app_token,
client_secret=mkm_app_secret,
resource_owner_key=mkm_access_token,
resource_owner_secret=mkm_token_secret,
realm=url
)
Other things I have run into in the past include the fact that the mkm api doesn't (or at least didn't) accept URI-escaped parameters, so you may need to unescape them.
For anyone who's reading in 2020, there's no need to reinvent the wheel, just pass the Oauth header and the parameters to requests, here's an example with metaproducts/find:
import requests
from requests_oauthlib import OAuth1
import json
import passwords
card_name = 'Tarmogoyf'
output = 'output.json'
base_url = 'https://api.cardmarket.com/ws/v2.0/' + output + '/'
url = base_url + 'metaproducts/find'
params={'search': card_name}
headeroauth = OAuth1(
realm = url,
client_key = passwords.mkm_app_token,
client_secret = passwords.mkm_app_secret,
resource_owner_key = passwords.mkm_access_token,
resource_owner_secret = passwords.mkm_token_secret,
)
response = requests.get(
url,
params,
auth = headeroauth
)
if (response.ok == True):
json_response = response.json()
print(json.dumps(json_response, indent=4, sort_keys=True))
else:
print(str(response.status_code) + " " + response.reason)
exit()
The /output.json/ part of the string makes the output JSON instead of XML

Categories

Resources