Using the TinEye API requires setting a custom multipart boundary on requests. I can do that with urllib3, but I'd prefer doing that with Python requests. For urllib3, calling the TinEye API looks like that:
from hashlib import sha1
from PIL import Image
import hmac, json, mimetools, random, string, time, urllib, urllib3
def get_tineye_results(path):
TINEYE_API_URL = 'http://api.tineye.com/rest/search/'
TINEYE_PUBLIC_KEY = 'your-public-key'
TINEYE_SECRET_KEY = 'your-secret-key'
filename = path.replace('\\', '/').rsplit('/', 1)[1]
t = int(time.time())
nonce = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(10)) # 8+ random characters
boundary = mimetools.choose_boundary()
to_sign = TINEYE_SECRET_KEY + 'POST' + 'multipart/form-data; boundary=' + boundary + urllib.quote_plus(filename) + str(t) + nonce + TINEYE_API_URL
signature = hmac.new(TINEYE_SECRET_KEY, to_sign, sha1).hexdigest()
data = { 'api_key': TINEYE_PUBLIC_KEY, 'date': t, 'nonce': nonce, 'api_sig': signature }
r = urllib3.connection_from_url(TINEYE_API_URL).request_encode_body('POST', TINEYE_API_URL+'?'+ urllib.urlencode(data), fields={'image_upload': (filename, open(path, 'rb').read())}, multipart_boundary=boundary)
return json.loads(r.data)
print get_tineye_results('/temp/my_image.jpg')
Problem is: I can't figure out a way to set the custom boundary in Python requests. There is an additional package for Python requests that supposedly allows this.
But I prefer to do this only in Python requests, or with Python standard libs urllib + urllib2.
Related
I am using the following examples from the docusign site.
I have a set of python scripts that works well on my PC.
I have the move the code to a server behind a proxy.
I could not find any example or settings to configure a proxy.
I tired setting it in the underlining URLLIB3 code but it is being overwritten each time the AP creates class of the APIClient().
How do I set the python docusign_esign ApiClient to use a proxy?
Below is the portion of the code.
from docusign_esign import ApiClient
from docusign_esign import EnvelopesApi
from jwt_helper import get_jwt_token, get_private_key
# this one has all the connection parameters
from jwt_config import DS_JWT
import urllib3
proxy = urllib3.ProxyManager('http://<id>:<pwd>#<proxy_server>:3128/', maxsize=10)
# used by docusign to decide what you have access to
SCOPES = ["signature", "impersonation"]
# Call the envelope status change method to list the envelopes changed in the last 10 days
def worker(args):
api_client = ApiClient()
api_client.host = args['base_path']
api_client.set_default_header("Authorization", "Bearer " + args['access_token'])
envelope_api = EnvelopesApi(api_client)
# The Envelopes::listStatusChanges method has many options
# The list status changes call requires at least a from_date OR
# a set of envelopeIds. Here we filter using a from_date.
# Here we set the from_date to filter envelopes for the last month
# Use ISO 8601 date format
from_date = (datetime.datetime.utcnow() - timedelta(days=120)).isoformat()
results = envelope_api.list_status_changes(args['account_id'], from_date=from_date)
return results, envelope_api
# Call request_jwt_user_token method
def get_token(private_key, api_client):
token_response = get_jwt_token(private_key, SCOPES, DS_JWT["authorization_server"],
DS_JWT["ds_client_id"], DS_JWT["ds_impersonated_user_id"])
access_token = token_response.access_token
# Save API account ID
user_info = api_client.get_user_info(access_token)
accounts = user_info.get_accounts()
api_account_id = accounts[0].account_id
base_path = accounts[0].base_uri + "/restapi"
return {"access_token": access_token, "api_account_id": api_account_id, "base_path":
base_path}
# bucket to keep track of token info
def get_args(api_account_id, access_token, base_path):
args = {
"account_id": api_account_id,
"base_path": base_path,
"access_token": access_token
}
return args
# start the actual code here create and then setup the object
api_client = ApiClient()
api_client.set_base_path(DS_JWT["authorization_server"])
api_client.set_oauth_host_name(DS_JWT["authorization_server"])
api_client.rest_client.pool_manager.proxy = proxy
api_client.rest_client.pool_manager.proxy.scheme = "http"
private_key = get_private_key(DS_JWT["private_key_file"]).encode("ascii").decode("utf-8")
jwt_values = get_token(private_key, api_client)
args = get_args(jwt_values["api_account_id"], jwt_values["access_token"], jwt_values["base_path"])
account_id = args["account_id"]
# return the envelope list and api_client object created to get it
results, envelope_api = worker(args)
print("We found " + str(results.result_set_size) + " sets of files")
for envelope in results.envelopes:
envelope_id = envelope.envelope_id
print("Extracting " + envelope_id)
# The SDK always stores the received file as a temp file you can not set the path for this
# Call the envelope get method
temp_file = envelope_api.get_document(account_id=account_id, document_id="archive",
envelope_id=envelope_id)
if temp_file:
print("File is here " + temp_file)
with zipfile.ZipFile(temp_file, 'r') as zip_ref:
zip_ref.extractall(extract_dir + envelope_id + "\\")
zip_ref.close()
print("Done extracting " + envelope_id + " deleting zip file")
os.remove(temp_file)
print("Deleted file here " + temp_file)
else:
print("Failed to get data for " + envelope_id)
I am using requests library (python 3.9) to get filename from URL.[1] For some reason a file name is incorrectly encoded.
I should get "Ogłoszenie_0320.pdf" instead of "OgÅ\x82oszenie_0320.pdf".
My code looks something like this:
import requests
import re
def getFilenameFromRequest(url : str, headers):
# Parses from header information
contentDisposition = headers.get('content-disposition')
if contentDisposition:
filename = re.findall('filename=(.+)', contentDisposition)
print("oooooooooo: " + contentDisposition + " : " + str(filename))
if len(filename) != 0:
return filename[0]
# Parses from url
parsedUrl = urlparse(url)
return os.path.basename(parsedUrl.path)
def getFilenameFromUrl(url : str):
request = requests.head(url)
headers = request.headers
return getFilenameFromRequest(url, headers)
getFilenameFromUrl('https://przedszkolekw.bip.gov.pl'+
'/fobjects/download/880287/ogloszenie-uzp-nr-613234-pdf.html')
Any idea how to fix it?
I know for standard request I can set encoding directly:
request.encoding = 'utf-8'
But what am I supposed to do with this case?
[1]
https://przedszkolekw.bip.gov.pl/fobjects/download/880287/ogloszenie-uzp-nr-613234-pdf.html
Only characters from the ascii based latin-1 should be used as header values [rfc]. Here the file name has been escaped.
>>> s = "Ogłoszenie_0320.pdf"
>>> s.encode("utf8").decode("unicode-escape")
'OgÅ\x82oszenie_0320.pdf'
To reverse the process you can do
>>> sx = 'OgÅ\x82oszenie_0320.pdf'
>>> sx.encode("latin-1").decode("utf8")
'Ogłoszenie_0320.pdf'
(updated after conversation in comments)
I am trying to retrieve Evernote OAuth access token through Python by following the documentation here. However, multiple attempts, I am not able to retrieve the temporary access token which is the very first step in this 3 legged authorization.
Any idea what am I doing wrong here?
import time
import base64
import random
import uuid
import urllib
import collections
import urllib.parse
import hmac
import hashlib
import binascii
import requests
def escape(s):
return urllib.parse.quote(s, safe='~')
def get_nonce():
return uuid.uuid4().hex
def stringify_parameters(parameters):
output = ''
ordered_parameters = {}
ordered_parameters =
collections.OrderedDict(sorted(parameters.items()))
counter = 1
for k, v in ordered_parameters.items():
output += escape(str(k)) + '=' + escape(str(v))
if counter < len(ordered_parameters):
output += '&'
counter += 1
return output
oauth_parameters={
'oauth_timestamp': str(int(time.time())),
'oauth_signature_method': "HMAC-SHA1",
'oauth_version': "1.0",
'oauth_nonce': get_nonce(),
'oauth_consumer_key': 'consumerkey',
'oauth_callback':'http://localhost'
}
string_parameters=stringify_parameters(oauth_parameters)
secret='secret'
signature = hmac.new(secret.encode(), string_parameters.encode(),hashlib.sha1).digest()
oauth_parameters['oauth_signature']=escape(base64.b64encode(signature).decode())
res=requests.get('https://sandbox.evernote.com/oauth?'+stringify_parameters(oauth_parameters))
print(res.status_code)
I think the way you create a signature is incorrect. This works for me:
key = (escape(secret)+'&').encode()
message = ('GET&' + escape('https://sandbox.evernote.com/oauth') + '&' + escape(string_parameters)).encode()
signature = hmac.new(key, message, hashlib.sha1).digest()
oauth_parameters['oauth_signature'] = base64.b64encode(signature).decode()
res = requests.get('https://sandbox.evernote.com/oauth?' + stringify_parameters(oauth_parameters))
I have the following code:
#!/usr/bin/python
import time, uuid, hmac, hashlib, base64, json
import urllib3
import certifi
import datetime
import requests
import re
from datetime import datetime
http = urllib3.PoolManager(
cert_reqs='CERT_REQUIRED', # Force certificate check.
ca_certs=certifi.where(), # Path to the Certifi bundle.
)
#Get the status response from pritunl api
BASE_URL = 'https://www.vpn.trimble.cloud:443'
API_TOKEN = 'gvwrfQZQPryTbX3l03AQMwTyaE0aFywE'
API_SECRET = 'B0vZp5dDyOrshW1pmFFjAnIUyeGtFy9y'
LOG_PATH = '/var/log/developer_vpn/'
def auth_request(method, path, headers=None, data=None):
auth_timestamp = str(int(time.time()))
auth_nonce = uuid.uuid4().hex
auth_string = '&'.join([API_TOKEN, auth_timestamp, auth_nonce,
method.upper(), path] + ([data] if data else []))
auth_signature = base64.b64encode(hmac.new(
API_SECRET, auth_string, hashlib.sha256).digest())
auth_headers = {
'Auth-Token': API_TOKEN,
'Auth-Timestamp': auth_timestamp,
'Auth-Nonce': auth_nonce,
'Auth-Signature': auth_signature,
}
if headers:
auth_headers.update(headers)
return http.request(method, BASE_URL + path, headers=auth_headers, body=data)
response1 = auth_request('GET',
'/server',
)
if response1.status == 200:
pritunlServResponse = (json.loads(response1.data))
#print pritunlServResponse
#print response1.data
Name = [y['name'] for y in pritunlServResponse]
Server_id = [x['id'] for x in pritunlServResponse]
for srv_name, srv_id in zip(Name, Server_id):
response2 = auth_request('GET',
'/server/' + srv_id + '/output',
)
pritunlServResponse2 = (json.loads(response2.data))
py_pritunlServResponse2 = pritunlServResponse2['output']
print("value of srv_id: ", srv_id, "\n")
print("value of srv_name: ", srv_name, "\n")
logfile = open(LOG_PATH + srv_name +'_vpn_out.log', 'w')
for log in py_pritunlServResponse2:
if re.search(r'(?!52\.39\.62\.8)', log):
logfile.write("%s\n" % log)
logfile.close()
else:
raise SystemExit
This code visits a website using authentication (the address has been redacted), grabs some text formatted in JSON, and parses two values from the output: "srv_name" and "srv_id". This code then uses the "srv_id" to construct additional HTTP requests to get log files from the server. It then grabs the log files - one for each "srv_id" and names them with the values obtained from "srv_name" and saves them on the local system.
I want to do some additional grep-style processing before the files are written to the local system. Specifically I'd like to exclude any text exactly containing "52.39.62.8" from being written. When I run the code above, it looks like the regex is not being processed as I still see "52.39.62.8" in my output files.
If the IP address is always flanked by specific characters, e.g.: (52.39.62.8):, you can use in for exact contains:
if '(52.39.62.8):' not in log:
logfile.write(log + '\n')
re.search(r'(?!52\.39\.62\.8)', log)
You're matching any empty string that is not followed by the ip address - every string will match, as this will match the end of any string.
reverse your logic and output the line to the log only if re.search for the ip address comes back as None.
if re.search(r'(?<!\d)52\.39\.62\.8(?!\d)', log) is None:
logfile.write("%s\n" % log)
note that this also includes it's own negative look-behind and look-ahead assertions to ensure no digits precede or follow the ip address.
I'm trying to get magiccardmarket.eu API authentication to work in Python, but no matter whether I'm using rauth or requests_oauthlib, I get 403.
My code is:
#!/usr/bin/python
import logging
import rauth
import requests_oauthlib
logging.basicConfig(level=logging.DEBUG)
mkm_app_token = 'B7VI9Qg2xh855WtR'
mkm_app_secret = '<cut>'
mkm_access_token = 'LQj2rUwOFUJsmuJvCTlny1UzGZSXzHjo'
mkm_token_secret = '<cut>'
url = 'https://sandbox.mkmapi.eu/ws/v1.1/account'
# session = rauth.OAuth1Session(
# consumer_key=mkm_app_token,
# consumer_secret=mkm_app_secret,
# access_token=mkm_access_token,
# access_token_secret=mkm_token_secret,
# )
session = requests_oauthlib.OAuth1Session(
mkm_app_token,
client_secret=mkm_app_secret,
resource_owner_key=mkm_access_token,
resource_owner_secret=mkm_token_secret,
)
r = session.get(url)
print(r)
When I look at debugging info, everything seems fine (of course besides 403 response):
DEBUG:requests_oauthlib.oauth1_auth:Signing request <PreparedRequest [GET]> using client <Client nonce=None, signature_method=HMAC-SHA1, realm=None, encoding=utf-8, timestamp=None, resource_owner_secret=****, decoding=utf-8, verifier=None, signature_type=AUTH_HEADER, rsa_key=None, resource_owner_key=LQj2rUwOFUJsmuJvCTlny1UzGZSXzHjo, client_secret=****, callback_uri=None, client_key=B7VI9Qg2xh855WtR>
DEBUG:requests_oauthlib.oauth1_auth:Including body in call to sign: False
DEBUG:oauthlib.oauth1.rfc5849:Collected params: [(u'oauth_nonce', u'87129670621454425921416648590'), (u'oauth_timestamp', u'1416648590'), (u'oauth_consumer_key', u'B7VI9Qg2xh855WtR'), (u'oauth_signature_method', u'HMAC-SHA1'), (u'oauth_version', u'1.0'), (u'oauth_token', u'LQj2rUwOFUJsmuJvCTlny1UzGZSXzHjo')]
DEBUG:oauthlib.oauth1.rfc5849:Normalized params: oauth_consumer_key=B7VI9Qg2xh855WtR&oauth_nonce=87129670621454425921416648590&oauth_signature_method=HMAC-SHA1&oauth_timestamp=1416648590&oauth_token=LQj2rUwOFUJsmuJvCTlny1UzGZSXzHjo&oauth_version=1.0
DEBUG:oauthlib.oauth1.rfc5849:Normalized URI: https://sandbox.mkmapi.eu/ws/v1.1/account
DEBUG:oauthlib.oauth1.rfc5849:Base signing string: GET&https%3A%2F%2Fsandbox.mkmapi.eu%2Fws%2Fv1.1%2Faccount&oauth_consumer_key%3DB7VI9Qg2xh855WtR%26oauth_nonce%3D87129670621454425921416648590%26oauth_signature_method%3DHMAC-SHA1%26oauth_timestamp%3D1416648590%26oauth_token%3DLQj2rUwOFUJsmuJvCTlny1UzGZSXzHjo%26oauth_version%3D1.0
DEBUG:oauthlib.oauth1.rfc5849:Signature: 291LTesHZR6W4bjZ1NqSW5hEgoM=
DEBUG:oauthlib.oauth1.rfc5849:Encoding URI, headers and body to utf-8.
DEBUG:requests_oauthlib.oauth1_auth:Updated url: https://sandbox.mkmapi.eu/ws/v1.1/account
DEBUG:requests_oauthlib.oauth1_auth:Updated headers: {'Accept': '*/*', 'Connection': 'keep-alive', 'Accept-Encoding': 'gzip, deflate', 'Authorization': 'OAuth oauth_nonce="87129670621454425921416648590", oauth_timestamp="1416648590", oauth_version="1.0", oauth_signature_method="HMAC-SHA1", oauth_consumer_key="B7VI9Qg2xh855WtR", oauth_token="LQj2rUwOFUJsmuJvCTlny1UzGZSXzHjo", oauth_signature="291LTesHZR6W4bjZ1NqSW5hEgoM%3D"', 'User-Agent': 'python-requests/2.4.3 CPython/2.7.8 Darwin/14.0.0'}
DEBUG:requests_oauthlib.oauth1_auth:Updated body: None
INFO:requests.packages.urllib3.connectionpool:Starting new HTTPS connection (1): sandbox.mkmapi.eu
DEBUG:requests.packages.urllib3.connectionpool:"GET /ws/v1.1/account HTTP/1.1" 403 None
This is not an issue of authentication details, which are provided on account profile page when you request dedicated application API access, since those details work fine with PHP example provided by the site: https://www.mkmapi.eu/ws/documentation/API:Auth_libcurl
When I go through site's documentation, nothing seems out of ordinary: https://www.mkmapi.eu/ws/documentation/API:Auth_Overview
I honestly don't know where to go from here...
I realized that the code above with requests_oauthlib didn't build the header like it was layed out in the documentation, so I ended up inventing the wheel again and building the header myself, following the steps outlined in the documentation: https://www.mkmapi.eu/ws/documentation/API:Auth_OAuthHeader
The following script is not very beautiful, but it does its job.
import requests
from urllib import quote_plus as rawurlencode
import time
import string
import random
import operator
from hashlib import sha1
from hmac import new as hmac
def id_generator(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
# personal Info - taken from https://www.mkmapi.eu/ws/documentation/API:Auth_Overview
mkmAppToken = 'bfaD9xOU0SXBhtBP'
mkmAppSecret = 'pChvrpp6AEOEwxBIIUBOvWcRG3X9xL4Y'
mkmAccessToken = 'lBY1xptUJ7ZJSK01x4fNwzw8kAe5b10Q'
mkmAccessSecret = 'hc1wJAOX02pGGJK2uAv1ZOiwS7I9Tpoe'
# Url to access on mkm
# note that this deviates from the example in the header documentation (https://www.mkmapi.eu/ws/documentation/API:Auth_OAuthHeader) which uses
#accessUrl = 'https://www.mkmapi.eu/ws/v1.1/account'
accessUrl = 'https://www.mkmapi.eu/ws/v1.1/output.json/account'
#Method for access
MyMethod = "GET"
baseString = MyMethod + "&" + rawurlencode(accessUrl) + "&"
# create a random string
# the documentation in https://www.mkmapi.eu/ws/documentation/API:Auth_OAuthHeader uses
#nonce = 53eb1f44909d6
nonce = id_generator(8)
# what time is it?
# the documentation in https://www.mkmapi.eu/ws/documentation/API:Auth_OAuthHeader uses
#now = 1407917892
now = str(int(time.time()))
MyOauthmethod = "HMAC-SHA1"
MyOauthver = "1.0"
# define Parameters and values, order doesn't matter
paramDict ={"oauth_consumer_key":mkmAppToken, "oauth_token" :mkmAccessToken, "oauth_nonce":nonce, "oauth_timestamp":now, "oauth_signature_method":MyOauthmethod, "oauth_version":MyOauthver}
# sorting of parameters is done here
sorted_paramDict = sorted(paramDict.items(), key=operator.itemgetter(0))
#collect the full parameters string
paramStr = ''
for kv in sorted_paramDict:
paramStr = paramStr + kv[0] + "=" + kv[1] + "&"
# and get rid of the trailing ampersand
paramStr = paramStr[:-1]
#concatenate request and oauth parameters
baseString = baseString + rawurlencode(paramStr)
# concatenate both keys
signingKey = rawurlencode(mkmAppSecret) + "&" + rawurlencode(mkmAccessSecret)
# and create a hased signature with the key and the baseString
Signature = hmac(signingKey, baseString, sha1).digest().encode('base64')[:-1]
# construct the header from the parameters and the URL and the signature
MyHeader = 'OAuth ' + 'realm="' + accessUrl + '", '
for kv in sorted_paramDict:
MyHeader += kv[0] + '="' + kv[1] + '",'
MyHeader += 'oauth_signature="' + Signature +'"'
headers = {'Authorization': MyHeader}
# and now requests can do its magic (pun intended)
r = requests.get(accessUrl, headers=headers)
outjson = r.json()
You need to provide the realm as an argument to the OAuth1Session, like so:
session = requests_oauthlib.OAuth1Session(
mkm_app_token,
client_secret=mkm_app_secret,
resource_owner_key=mkm_access_token,
resource_owner_secret=mkm_token_secret,
realm=url
)
Other things I have run into in the past include the fact that the mkm api doesn't (or at least didn't) accept URI-escaped parameters, so you may need to unescape them.
For anyone who's reading in 2020, there's no need to reinvent the wheel, just pass the Oauth header and the parameters to requests, here's an example with metaproducts/find:
import requests
from requests_oauthlib import OAuth1
import json
import passwords
card_name = 'Tarmogoyf'
output = 'output.json'
base_url = 'https://api.cardmarket.com/ws/v2.0/' + output + '/'
url = base_url + 'metaproducts/find'
params={'search': card_name}
headeroauth = OAuth1(
realm = url,
client_key = passwords.mkm_app_token,
client_secret = passwords.mkm_app_secret,
resource_owner_key = passwords.mkm_access_token,
resource_owner_secret = passwords.mkm_token_secret,
)
response = requests.get(
url,
params,
auth = headeroauth
)
if (response.ok == True):
json_response = response.json()
print(json.dumps(json_response, indent=4, sort_keys=True))
else:
print(str(response.status_code) + " " + response.reason)
exit()
The /output.json/ part of the string makes the output JSON instead of XML