I currently have this class for making requests to an API and caching the JSON response:
import os
import pathlib
import json
import hashlib
import time
import requests
class NoJSONResponseError(Exception):
pass
class JSONRequestCacher(object):
"""Manage a JSON object through the cache.
Download the associated resource from the provided URL
when need be and retrieve the JSON from a cached file
if possible.
"""
def __init__(self, duration=86400, cachedir=None):
self.duration = duration
self.cachedir = self._get_cachedir(cachedir)
self._validate_cache()
def _get_cachedir(self, cachedir):
if cachedir is None:
cachedir = os.environ.get(
'CUSTOM_CACHEDIR',
pathlib.Path(pathlib.Path.home(), '.custom_cache/')
)
return cachedir
def _validate_cache(self):
"""Create the cache directory if it doesn't exist"""
self.cachedir.mkdir(parents=True, exist_ok=True)
def _request(self, url):
"""Perform the retrieval of the requested JSON data"""
return requests.get(url)
def save(self, raw, cachefile):
"""Save the provided raw JSON data into the cached file"""
with open(cachefile, 'w') as out:
json.dump(raw, out)
def load(self, cachefile):
"""Retrieve the saved JSON data from the cached file"""
with open(cachefile) as cached:
return json.load(cached)
def cache_is_valid(self, cachefile):
"""Check if cache exists and is more recent than the cutoff"""
if cachefile.is_file():
cache_age = time.time() - cachefile.stat().st_mtime
return cache_age < self.duration
return False
def request(self, url, refresh=False):
"""The JSON data associated to the given URL.
Either read from the cache or fetch from the web.
"""
urlhash = hashlib.md5(url.encode()).hexdigest()
cachefile = self.cachedir.joinpath(urlhash)
start = time.time()
if not refresh and self.cache_is_valid(cachefile):
return self.load(cachefile), True, time.time() - start
resp = self._request(url)
resp.raise_for_status()
try:
raw = resp.json()
except ValueError:
raise NoJSONResponseError()
self.save(raw, cachefile)
return raw, False, resp.elapsed.total_seconds()
I then have other classes and code which call the request method of this code like so:
class APIHelper():
def __init__(self):
self.cache = JSONRequestCacher()
def fetch(val):
url = 'my/url/{}'.format(val)
return self.cache.request(url)
def fetchall(vals):
repsonses = []
for val in vals:
responses.append(self.fetch(val))
return responses
For a small number of vals this is fine and it's really no big deal to wait 10 mins. However I am now looking at making 30,000+ hits to this endpoint. In the past I have used threadpools (multiprocessing.dummy.Pool) to achieve some parallelism, however from my reading it seems like async/await and aiothttp is a better way to go. Unfortunately try as I might I cannot wrap my head around how to translate that to this code. I am using Python 3.8.
EDIT
I tried making this change:
class JSONRequestCacher():
def __init__():
self.http = aiohttp.ClientSession()
async def _request(self, url):
async with self.http.get(url) as response:
return await response.read()
Got the error: AttributeError: 'coroutine' object has no attribute 'json' from my raw = resp.json() line
Tried then adding resp = await self._request(url) but that is SyntaxError: 'await' outside async function. Then if I make request an async function then calling it just seems to return me a coroutine object that doesn't give me the expected response.
And this is just trying to make the _request call async. I can't even start to understand how I am meant to make multiple calls to it via another class (APIHelper).
Related
From the docs at https://docs.python-requests.org/en/master/user/authentication/
I gathered that the __call__ function in my own Auth Class should have the r argument,
However when i go to call this class in requests.get(auth=MyClass), I get the error TypeError: __call__() missing 1 required positional argument: 'r'
The code for my class can be found here https://pastebin.com/YDZ2DeaT
import requests
import time
import base64
from requests.auth import AuthBase
class TokenAuth(AuthBase):
"""Refreshes SkyKick token, for use with all Skykick requests"""
def __init__(self, Username: str, SubKey: str):
self.Username = Username
self.SubKey = SubKey
# Initialise with no token and instant expiry
self.Token = None
self.TokenExpiry = time.time()
self.Headers = {
# Request headers
'Content-Type' : 'application/x-www-form-urlencoded',
'Ocp-Apim-Subscription-Key': self.SubKey,
}
self.Body = {
# Request body
'grant_type': 'client_credentials',
'scope' : 'Partner'
}
def regenToken(self):
# Sends request to regenerate token
try:
# Get key from API
response = requests.post("https://apis.skykick.com/auth/token",
headers=self.Headers,
auth=(self.Username, self.SubKey),
data=self.Body,
).json()
except:
raise Exception("Sending request failed, check connection.")
# API errors are inconsistent, easiest way to catch them
if "error" in response or "statusCode" in response:
raise Exception(
"Token requesting failed, cannot proceed with any Skykick actions, exiting.\n"
f"Error raised was {response}")
# Get token from response and set expiry
self.Token = response["access_token"]
self.TokenExpiry = time.time() + 82800
def __call__(self, r):
# If token expiry is now or in past, call regenToken
if self.TokenExpiry <= time.time():
self.regenToken()
# Set headers and return complete requests.Request object
r.headers["Authorization"] = f"Bearer {self.Token}"
return r
# Initialise our token class, so it is ready to call
TokenClass = TokenAuth("test", "1234")
#Send request with class as auth method.
requests.get("https://apis.skykick.com/whoami", auth=TokenClass())
I've tried using the example code, which works, but I can't figure out why mine won't work.
python-requests version is 2.25.1
I think I know what is going on.
This line instantiates an object, called TokenClass
TokenClass = TokenAuth("test", "1234")
then here,
requests.get("https://apis.skykick.com/whoami", auth=TokenClass())
you are calling that object like a function
when you call an object like a function, python looks for the __call__ method of the object.
And you are not calling in any arguments here. What you have is roughly the same as this I think
requests.get("https://apis.skykick.com/whoami", auth=TokenClass.__call__())
and so it complains that you are missing the r argument
This is their example:
import requests
class MyAuth(requests.auth.AuthBase):
def __call__(self, r):
# Implement my authentication
return r
url = 'https://httpbin.org/get'
requests.get(url, auth=MyAuth())
MyAuth is a class that they define, and then MyAuth() creates an instance of it that they pass in to get.
Yours is more like this
import requests
class MyAuth(requests.auth.AuthBase):
def __call__(self, r):
# Implement my authentication
return r
url = 'https://httpbin.org/get'
myAuth = MyAuth() # create an instance of the class
requests.get(url, auth=myAuth()) # call the instance and pass in result
It could also be written like this
import requests
class MyAuth(requests.auth.AuthBase):
def __call__(self, r):
# Implement my authentication
return r
url = 'https://httpbin.org/get'
requests.get(url, auth=MyAuth()())
This program with produce the same error you are getting
import requests
class MyAuth(requests.auth.AuthBase):
def __call__(self, r):
# Implement my authentication
return r
url = 'https://httpbin.org/get'
MyAuth()()
because when you put () after a class, you get an instance, and when you put () after an instance, you call the __call__ method
I am trying to use tornado to do a simple get and post method. Quite new to tornado framework. For the post I would like to take in a json as input, use that input to feed into another function that I have to execute another part of code. However I can't get tornado post method to work even with a simple self.write().
For my get method I am reading from an SQL database to get the status of a sensor and write that in a json format. The get method works perfectly! When I go to localhost:port# it reads out my get json file. For my post method I would like to take in a simple json of just one key:value which is a float number. I want to take that float number that the user specified in the json and use it in my flowMKS.set() function that will change the setpoint parameter of the sensor. I am not sure how to input a json into the post method and read it into a variable. I have some #commented code below that I tried and didn't work. However I went back to the basics and just did a self.write("Hello World") to see if the post was working. I can't get self.write to work either. Keep getting a 500 error message when i go to localhost:port#/flow_post. The variable flow_status was used in my get method.
The intended result would be to take in a json {"setpoint":45.5} into the post method. Use the number and insert into my flowMKS method to change a parameter on the sensor.
How would you take in a json to a post method and take the number from the json input and store in a variable?
class Current(tornado.web.RequestHandler):
def get(self):
global flow_status
time = flow_status[0]
ip = flow_status[1]
rate = flow_status[2]
setp = flow_status[3]
tempc = flow_status[4]
status = {"flow_controller":{
"time":time,
"ip":ip,
"rate_sccm":rate,
"setpoint":setp,
"temperature_c":tempc,
}
}
self.write(status)
class Update(tornado.web.RequestHandler):
# def prepare(self):
# if self.request.haders["Content-Type"].startswith("application/json"):
# self.json_args = json.loads(self.request.body)
# else:
# self.json_args = None
def post(self):
# #expecting body data to contain JSON so we use json.loads to decrypt the JSON into a dict
# data = json.loads(self.request.body)
#
# #Getting what the setpoint should be
# setpoint = self.json_args["setpoint"]
#
# #making the input a float
# setpoint = float(setpoint)
#
# #setting up connection with sensor
# flowMKS = FlowController(flow_status[1])
#
# #sending setpoint to sensor
# flowMKS.set(setpoint)
self.write("Hello World")
if __name__ == '__main__':
# global flow_status
#Below is creating the Tornado based API for get and post methods
tornado.options.parse_command_line()
app = tornado.web.Application(
handlers=[(r'/',Current), (r'/flow_post', Update)])
http_server = tornado.httpserver.HTTPServer(app)
http_server.listen(options.port)
#using PeriodicCallback to get info from the SQL database every 500 ms
PeriodicCallback(get_sql_status,500).start()
#starting the entire Tornado IOLoop
tornado.ioloop.IOLoop.current().start()
For uploading a file using Tornado you can use this function tornado.httputil.parse_body_arguments which will split the uploaded file content in a dictionary file_dict and other arguments in the FormData in the args_dict.
Sample code:
import tornado.httputil
import tornado.web
import tornado.escape
import json
import os
import sys
import traceback
class FileHandler(tornado.web.RequestHandler):
def _return_response(self, request, message_to_be_returned: dict, status_code):
"""
Returns formatted response back to client
"""
try:
request.set_header("Content-Type", "application/json; charset=UTF-8")
request.set_status(status_code)
#If dictionary is not empty then write the dictionary directly into
if(bool(message_to_be_returned)):
request.write(message_to_be_returned)
request.finish()
except Exception:
raise
def set_default_headers(self, *args, **kwargs):
self.set_header('Content-Type','text/csv')
self.set_header("Access-Control-Allow-Origin", "*")
self.set_header("Access-Control-Allow-Headers", "x-requested-with")
self.set_header("Access-Control-Allow-Methods", "*")
def post(self):
"""
This function reads an uploaded file
"""
try:
arg_dict = {}
file_dict = {}
tornado.httputil.parse_body_arguments(self.request.headers["Content-Type"], self.request.body, arg_dict, file_dict)
uploaded_file = file_dict['TestFile'][0]
if not uploaded_file:
return self._return_response(self, { 'message': 'No test file uploaded, please upload a test file' }, 400)
# File contents here
file_contents = str(uploaded_file['body'], "utf-8")
self.set_status(200)
self.finish()
except Exception as ex:
return self._return_response(self, { "message": 'Could not complete the request because of some error at the server!', "cause": ex.args[0], "stack_trace": traceback.format_exc(sys.exc_info()) }, 500)
You can alternatively use tornado.escape.json_decode to deserialize the request body into a dictionary and do something with it.
Sample code:
import tornado.gen
import tornado.web
import tornado.escape
import json
import os
import sys
import traceback
class JSONHandler(tornado.web.RequestHandler):
def _return_response(self, request, message_to_be_returned: dict, status_code):
"""
Returns formatted response back to client
"""
try:
request.set_header("Content-Type", "application/json; charset=UTF-8")
request.set_status(status_code)
#If dictionary is not empty then write the dictionary directly into
if(bool(message_to_be_returned)):
request.write(message_to_be_returned)
request.finish()
except Exception:
raise
def set_default_headers(self, *args, **kwargs):
self.set_header("Content-Type", "application/json")
self.set_header("Access-Control-Allow-Origin", "*")
self.set_header("Access-Control-Allow-Headers", "x-requested-with")
self.set_header("Access-Control-Allow-Methods", "*")
def post(self):
"""
This function parses the request body and does something
"""
try:
# Do something with request body
request_payload = tornado.escape.json_decode(self.request.body)
return self._return_response(self, request_payload, 200)
except json.decoder.JSONDecodeError:
return self._return_response(self, { "message": 'Cannot decode request body!' }, 400)
except Exception as ex:
return self._return_response(self, { "message": 'Could not complete the request because of some error at the server!', "cause": ex.args[0], "stack_trace": traceback.format_exc(sys.exc_info()) }, 500)
I am trying to understand how to handle a grpc api with bidirectional streaming (using the Python API).
Say I have the following simple server definition:
syntax = "proto3";
package simple;
service TestService {
rpc Translate(stream Msg) returns (stream Msg){}
}
message Msg
{
string msg = 1;
}
Say that the messages that will be sent from the client come asynchronously ( as a consequence of user selecting some ui elements).
The generated python stub for the client will contain a method Translate that will accept a generator function and will return an iterator.
What is not clear to me is how would I write the generator function that will return messages as they are created by the user. Sleeping on the thread while waiting for messages doesn't sound like the best solution.
This is a bit clunky right now, but you can accomplish your use case as follows:
#!/usr/bin/env python
from __future__ import print_function
import time
import random
import collections
import threading
from concurrent import futures
from concurrent.futures import ThreadPoolExecutor
import grpc
from translate_pb2 import Msg
from translate_pb2_grpc import TestServiceStub
from translate_pb2_grpc import TestServiceServicer
from translate_pb2_grpc import add_TestServiceServicer_to_server
def translate_next(msg):
return ''.join(reversed(msg))
class Translator(TestServiceServicer):
def Translate(self, request_iterator, context):
for req in request_iterator:
print("Translating message: {}".format(req.msg))
yield Msg(msg=translate_next(req.msg))
class TranslatorClient(object):
def __init__(self):
self._stop_event = threading.Event()
self._request_condition = threading.Condition()
self._response_condition = threading.Condition()
self._requests = collections.deque()
self._last_request = None
self._expected_responses = collections.deque()
self._responses = {}
def _next(self):
with self._request_condition:
while not self._requests and not self._stop_event.is_set():
self._request_condition.wait()
if len(self._requests) > 0:
return self._requests.popleft()
else:
raise StopIteration()
def next(self):
return self._next()
def __next__(self):
return self._next()
def add_response(self, response):
with self._response_condition:
request = self._expected_responses.popleft()
self._responses[request] = response
self._response_condition.notify_all()
def add_request(self, request):
with self._request_condition:
self._requests.append(request)
with self._response_condition:
self._expected_responses.append(request.msg)
self._request_condition.notify()
def close(self):
self._stop_event.set()
with self._request_condition:
self._request_condition.notify()
def translate(self, to_translate):
self.add_request(to_translate)
with self._response_condition:
while True:
self._response_condition.wait()
if to_translate.msg in self._responses:
return self._responses[to_translate.msg]
def _run_client(address, translator_client):
with grpc.insecure_channel('localhost:50054') as channel:
stub = TestServiceStub(channel)
responses = stub.Translate(translator_client)
for resp in responses:
translator_client.add_response(resp)
def main():
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
add_TestServiceServicer_to_server(Translator(), server)
server.add_insecure_port('[::]:50054')
server.start()
translator_client = TranslatorClient()
client_thread = threading.Thread(
target=_run_client, args=('localhost:50054', translator_client))
client_thread.start()
def _translate(to_translate):
return translator_client.translate(Msg(msg=to_translate)).msg
translator_pool = futures.ThreadPoolExecutor(max_workers=4)
to_translate = ("hello", "goodbye", "I", "don't", "know", "why",)
translations = translator_pool.map(_translate, to_translate)
print("Translations: {}".format(zip(to_translate, translations)))
translator_client.close()
client_thread.join()
server.stop(None)
if __name__ == "__main__":
main()
The basic idea is to have an object called TranslatorClient running on a separate thread, correlating requests and responses. It expects that responses will return in the order that requests were sent out. It also implements the iterator interface so that you can pass it directly to an invocation of the Translate method on your stub.
We spin up a thread running _run_client which pulls responses out of TranslatorClient and feeds them back in the other end with add_response.
The main function I included here is really just a strawman since I don't have the particulars of your UI code. I'm running _translate in a ThreadPoolExecutor to demonstrate that, even though translator_client.translate is synchronous, it yields, allowing you to have multiple in-flight requests at once.
We recognize that this is a lot of code to write for such a simple use case. Ultimately, the answer will be asyncio support. We have plans for this in the not-too-distant future. But for the moment, this sort of solution should keep you going whether you're running python 2 or python 3.
In Python I am trying to create an API for a connected device. I want to be available for both threaded (using request) and async applications (using aiohttp).
What I've come up with is wrapping the get method of both requests and aiohttp in a decorator. This decorator is passed at init and API calls are explicitly wrapped using the passed decorator.
It works, but I'd like to know how others think of this approach ? Are there better ways or will I be running into issues later on ?
Any help appreciated !
def threaded_gett(function):
# The threaded decorator
def wrapper(*args, **kwargs):
url, params = function(*args)
response = requests.get(url, params)
_json = response.json()
return function.__self__.process_response(_json)
return wrapper
def async_gett(function):
# The async decorator
def wrapper(*args, **kwargs):
url, params = function(*args)
try:
resp = yield from function.__self__.session.get(url, params=params)
except Exception as ex:
lgr.exception(ex)
else:
_json = yield from resp.json()
yield from resp.release()
return function.__self__.process_response(_json)
# wrapping the decorator in the async coroutine decorator.
wrapper = asyncio.coroutine(wrapper)
return wrapper
class ThreadedApi(BaseApi):
def __init__(self,threaded_gett):
Base.__init(self,threaded_gett)
class AsyncApi(BaseApi):
def __init__(self,async_gett):
Base.__init(self,async_gett)
class BaseApi():
def __init__(self,get_wrapper):
self.status = get_wrapper(self.status)
def status(self):
return <status path>
Your code is not complete but yes, the approach might work in simple cases (when .process_response() is very generic and could be applied to all API calls).
I've been playing with tweepy for a while, but I keep having rate-limiting issues, getting 429 errors. I know you can set the headers on individual calls like
api.get_user('twitter', headers={'User-Agent': 'MyUserAgent'})
but is there a way to set the header in one place and not have to do it on every api call?
Hacky way:
import functools
class NewAPI(object):
def __init__(self, api):
self.api = api
def __getattr__(self, key):
call = getattr(self.api, key)
#functools.wraps(call)
def wrapped_call(*args, **kwargs):
headers = kwargs.pop('headers', {})
headers['User-Agent'] = 'MyUserAgent' # or make this a class variable/instance variable
kwargs['headers'] = headers
return call(*args, **kwargs)
return wrapped_call
api = NewAPI(api)
print(api.get_user('twitter'))
Disclaimer: untested as I don't have tweepy.