I'm querying a db of election info and spitting them out in JSONP to use for some ajax/jQuery widgets we are creating.
What I'm wanting to do however is cache the view so that a new request isn't coming in every time the widget is rendered.
I'm running into 2 issues though.
Which method of caching should I use to render this view so that the database isn't getting hit hard.
I need to render the view as JSONP which means I need the callback variable to be dynamic (which it current'y is with my script but I'm afraid it wouldn't be with a cached view).
Here's what my code looks like.
from models import race
from models import candidates
from django.http import HttpResponse
from django.utils import simplejson
def data(request):
data = []
races = race.objects.all()
for election in races:
race_candidate = candidates.objects.filter(race__id=election.pk)
candidate_info = []
for n,candidate in enumerate(race_candidate):
candidate_values = {
"name":candidate.name,
"percent":candidate.percent,
"totalvotes":candidate.totalvotes,
"partyname":candidate.partyname,
"partyabbrv":candidate.partyabbrv,
}
candidate_info.append(candidate_values)
race_values = {
"title":election.title,
"description":election.description,
"priority":election.priority,
"decided":election.decided,
"candidates":candidate_info,
}
data.append(race_values)
json_races = '{"races":' + simplejson.dumps(data) + '}'
if("callback" in request.GET.keys()):
callback = request.GET["callback"]
else:
callback = None
if(callback):
response = HttpResponse("%s(%s)" % (
callback,
simplejson.dumps(data)
), mimetype="application/json"
)
else:
response = HttpResponse(json_races, mimetype="application/json")
return response
You'll have to use the low-level cache API with whichever cache backend you like.
from django.core.cache import cache
...
CACHE_TIMEOUT = 3600 # choose your timeout
def data(request):
jsonstring = cache.get('elections')
if jsonstring is None:
data = [{
"title": election.title,
"description": election.description,
"priority": election.priority,
"decided": election.decided,
"candidates": [
{
"name": candidate.name,
"percent": candidate.percent,
"totalvotes": candidate.totalvotes,
"partyname": candidate.partyname,
"partyabbrv": candidate.partyabbrv,
} for candidate in election.candidates_set.all()
],
} for election in race.objects.all()]
jsonstring = simplejson.dumps(data)
cache.set('elections', jsonstring, CACHE_TIMEOUT)
callback = request.GET.get('callback')
if callback:
response = HttpResponse("%s(%s)" % (callback, jsonstring),
mimetype="application/json")
else:
response = HttpResponse('{"races":' + jsonstring + '}',
mimetype="application/json")
return response
In your settings.py, configure CACHE according to documentation. The memory backend is the simplest, memcached is probably the best.
Related
I would like to write an api with django rest framework, I got some issues with my callback function.
I can get the access code, but how to give it to my app?
This is my callback function :
#api_view(['GET'])
def callback(request):
if request.method == 'GET':
code = request.GET.get("code")
encoded_credentials = base64.b64encode(envi.SECRET_ID.encode() + b':' + envi.SECRET_PASS.encode()).decode("utf-8")
token_headers = {
"Authorization": "Basic " + encoded_credentials,
"Content-Type": "application/x-www-form-urlencoded"
}
token_data = {
"grant_type": "authorization_code",
"code": code,
"redirect_uri": "http://127.0.0.1:800/callback"
}
test = "test :" + code
return JsonResponse(test, safe=False)
And this is my view where I try to do some stuff (I use spotify's API, with spotipy), I need to get the users name or mail :
#api_view(['GET'])
#permission_classes([permissions.IsAuthenticated])
def test(request):
if request.method == 'GET':
test = "test " + request.user.username
scope = "user-read-private"
sp = getScope(scope)
print(sp.current_user())
urn = 'spotify:artist:3jOstUTkEu2JkjvRdBA5Gu'
sp = spotipy.Spotify(auth_manager=SpotifyOAuth(client_id=envi.SECRET_ID, client_secret=envi.SECRET_PASS, redirect_uri=envi.SPOTIPY_REDIRECT_URI))
artist = sp.artist(urn)
print(artist)
user = sp.current_user()
return JsonResponse(user, safe=False)
def getScope(spotipyScope):
token = SpotifyOAuth(scope=spotipyScope,client_id=envi.SECRET_ID, client_secret=envi.SECRET_PASS, redirect_uri=envi.SPOTIPY_REDIRECT_URI)
spotifyObject = spotipy.Spotify(auth_manager= token)
return spotifyObject
When I do a get on 127.0.0.1:8000/test/, I have a new page on my browser, from spotify, I connect my account, and then, it redirects me on 127.0.0.1:8000/callback/?code=some_code
How can I give it to my first page waiting for the code so I can print the users stuff pls?
I have an api view which fetches data from its model. I want this api view to have an additional value from third party api. Here is views.py which grabs all of its model properties.
#api_view(
["GET"],
)
def business_asset_risk_details(request):
res = models.BusinessImpact.objects.all().values(
"hierarchy",
"business_assets",
"asset_name",
"vendors",
"product",
"version",
"cpe",
"asset_type",
"asset_categorization",
"_regulations",
"asset_risk",
)
return Response({"business_impact_details": res})
Here is third party api view fetching a specific values of it's own.
#api_view(
["GET"],
)
def cve_summery(request, key):
r = requests.get(
"https://services.nvd.nist.gov/rest/json/cves/1.0?cpeMatchString={}".format(
key)
)
if r.status_code == 200:
result = []
res = r.json().get("result").get("CVE_Items")
for rs in res:
data = {
"VulnID": rs.get("cve").get("CVE_data_meta").get("ID"),
"Summery": rs.get("cve").get("description").get("description_data"),
"exploitabilityScore": rs.get("impact")
.get("baseMetricV2")
.get("exploitabilityScore"),
"severity": rs.get("impact").get("baseMetricV2").get("severity"),
"impactScore": rs.get("impact").get("baseMetricV2").get("impactScore"),
}
result.append(data)
return Response(result)
return Response("error happend", r.status_code)
The logic here is, I want business_asset_risk_details view to return "severity": rs.get("impact").get("baseMetricV2").get("severity"), from cve_summery view.
NB: THE KEY parameter in cve_summery view IS cpe of business_asset_risk_details
Thanks.
Upon implementing a throttle for a REST API, I'm encountering an issue when running my tests all at once.
Upon isolating the subject TestCase and running the test runner, the TestCase passes its assertions. However when all the tests are ran I get the following error: AssertionError: 429 != 400. Which that type of error of course is due to the requests exceeding a rate limit.
How can I disable throttling for the tests so the assertion error is not raised. I decorated the TestCase with #override_settings but that doesn't have any effect.
from copy import deepcopy
from django.conf import settings
from django.test import TestCase, override_settings
from django.contrib.auth.models import User
from rest_framework.test import APITestCase, APIClient
from django.urls import reverse
from ..models import QuestionVote, Question
from users.models import UserAccount
from tags.models import Tag
from .model_test_data import mock_questions_submitted
REST_FRAMEWORK = deepcopy(settings.REST_FRAMEWORK)
del REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']
#override_settings(REST_FRAMEWORK=REST_FRAMEWORK)
class TestUserVoteOnOwnQuestion(APITestCase):
'''Verify that a User cannot vote on their own Question'''
#classmethod
def setUpTestData(cls):
cls.user1 = User.objects.create_user("Me", password="topsecretcode")
cls.user1_account = UserAccount.objects.create(user=cls.user1)
cls.tag = Tag.objects.create(name="Tag")
cls.q = mock_questions_submitted[2]
cls.q.update({'user_account': cls.user1_account})
cls.question = Question(**cls.q)
cls.question.save()
cls.question.tags.add(cls.tag)
def test_vote_on_own_posted_question(self):
self.client.login(username="Me", password="topsecretcode")
response = self.client.put(
reverse("questions_api:vote", kwargs={'id': 1}),
data={"vote": "upvote"}
)
self.assertEqual(response.status_code, 400)
self.assertEquals(
response.data['vote'],
"Cannot vote on your own question"
)
REST_FRAMEWORK = {
'TEST_REQUEST_DEFAULT_FORMAT': 'json',
'DEFAULT_THROTTLE_RATES': {
'voting': '5/minute'
}
}
class UserQuestionVoteView(APIView):
renderer_classes = [JSONRenderer, ]
parser_classes = [JSONParser, ]
permission_classes = [IsAuthenticated, ]
authentication_classes = [SessionAuthentication, ]
throttle_classes = [ScopedRateThrottle, ]
throttle_scope = "voting"
def put(self, request, id):
# import pdb; pdb.set_trace()
account = UserAccount.objects.get(user=request.user)
question = Question.objects.get(id=id)
if account == question.user_account:
return Response(data={
'vote': "Cannot vote on your own question"
}, status=400)
try:
stored_vote = QuestionVote.objects.get(
account=account, question=question
)
serializer = QuestionVoteSerializer(stored_vote, request.data)
except QuestionVote.DoesNotExist:
serializer = QuestionVoteSerializer(data=request.data)
finally:
if serializer.is_valid(raise_exception=True):
question_vote = serializer.save(
account=account,
question=question
)
vote = serializer.validated_data['vote']
if vote == "downvote":
question.vote_tally = F('vote_tally') - 1
else:
question.vote_tally = F('vote_tally') + 1
question.save()
question.refresh_from_db()
return Response(data={
'id': question.id,
'tally': question.vote_tally
})
return Response(serializer.errors)
One way to do this is by setting your config files up to support testing versions:
# config.py
REST_FRAMEWORK = {
'TEST_REQUEST_DEFAULT_FORMAT': 'json',
'DEFAULT_THROTTLE_RATES': {
'voting': '5/minute'
}
}
TESTING = len(sys.argv) > 1 and sys.argv[1] == 'test'
if TESTING:
del REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']
The pro of this approach is you're not hacking away at your application in tests and hiding modifications to the config file - all of your testing based changes are in the same config file as their true values.
The con of this approach is unless your developers know this setting and values are there, they may be scratching their heads as to why the throttling doesn't work in tests, but does at runtime.
My solution was to apply some monkey patching.
I have a throttles.py file where I have custom throttles, such as
class UserBurstRateThrottle(UserRateThrottle):
rate = '120/minute'
What I've done is create a stub allow_request function to always return true, so something like
def apply_monkey_patching_for_test():
def _allow_request(self, request, view):
return True
UserBurstRateThrottle.allow_request = _allow_request
Then, in the test_whatever.py file, I add the following at the top.
from my_proj import throttles
throttles.apply_monkey_patching_for_test()
Another easy way is to disable the cache that's responsible for storing the clients' meta data (https://www.django-rest-framework.org/api-guide/throttling/#setting-up-the-cache). So you need this in your test settings:
CACHES = {
'<throttling-cache-name|default>': {
'BACKEND': 'django.core.cache.backends.dummy.DummyCache', # to prevent API throttling
}
}
Status Quo:
Whenever a user visits my web application, Axios makes a request to a third party API to fetch data and populate the frontend with that data using v-for.
Conclusion: I have one API call per website visitor.
Desired status:
Whenever a user visits my web application, Axios shall fetch the data from the SQLite database which itself is populated every XX seconds by a python request to reduce API calls.
Questions:
Now I implemented a SQLite database using Django models and views. So far so good, the API gets fetched regularly and updates the database table properly.
1.) How can I now call the data in the database using Axios? As by my research Axios somehow needs to call a view and the view will call the data from the database, is this correct?
2.) If Axios needs to call a view, do I need another view.py file that calls the database? If I would insert the needed view function into the existing view.py file it would initiate another API call, wouldn't it?
3.) And how can I implement the link to the view function to Axios? Instead of a third party API url would I just use the path to the view file?
Quotes_app/Views.py:
from django.shortcuts import render
from Quotes_app.models import ratesEUR
import json
import requests
response = requests.get("http://data.fixer.io/api/latest?access_key=XXXX&base=EUR")
rates_EUR = json.loads(response.content.decode('utf-8'))
timestamp = rates_EUR['timestamp']
base = rates_EUR['base']
date = rates_EUR['date']
rates = rates_EUR['rates']
id = 1
rates_new = ratesEUR(id=id, timestamp=timestamp, base=base, date=date, rates=rates)
rates_new.save()
def render_Quotes_app(request, template="Quotes_app/templates/Quotes_app/Quotes_app.html"):
return render(request, template)
Quotes_app/models.py:
from django.db import models
class ratesEUR(models.Model):
timestamp = models.CharField(max_length=10)
base = models.CharField(max_length=3)
date = models.DateField(auto_now=False, auto_now_add=False)
rates = models.CharField(max_length=8)
def __str__(self):
return self.base
Vue.js/axios: (as of now directly fetching the API)
Vue.config.devtools = true;
var app = new Vue({
delimiters: ['[[', ']]'],
el: '.eurQuotesWrapper',
data() {
return {
rates: [],
};
},
computed: {
rates1() {
const ratesArr1 = Object.entries(this.rates);
const ret = ratesArr1.reduce((a, c, i, d) => {
if (i < d.length / 2) a[c[0]] = c[1];
return a;
}, {});
console.log('rates1', ret);
return ret;
},
rates2() {
const ratesArr2 = Object.entries(this.rates);
const ret = ratesArr2.reduce((a, c, i, d) => {
if (i >= d.length / 2) a[c[0]] = c[1];
return a;
}, {});
console.log('rates2', ret);
return ret;
}
},
created() {
axios
.get("http://data.fixer.io/api/latest?access_key=XXXX&base=EUR")
.then(response => {
this.rates = response.data.rates;
console.log(this.rates);
for(key in this.rates) {
this.rates[key] = new Intl.NumberFormat('de-DE', {
minimumFractionDigits: 5,
maximumFractionDigits: 5
}).format(this.rates[key]);
}
console.log(this.rates);
});
}
});
In advance thank you very much for your help!
You can use DRF(Django Rest Framework) to make REST API's or You can use JsonResponse
to send JSON objects as response.
from django.http import JsonResponse
data = # Your data
JsonResponse(data, encoder=MyJSONEncoder)
For more details visit django's documentation
Better would be if you use DRF for making rest apis. It's much easier.
I'm just wondering if there is any way to write a python script to check to see if a twitch.tv stream is live?
I'm not sure why my app engine tag was removed, but this would be using app engine.
Since all answers are actually outdated as of 2020-05-02, i'll give it a shot. You now are required to register a developer application (I believe), and now you must use an endpoint that requires a user-id instead of a username (as they can change).
See https://dev.twitch.tv/docs/v5/reference/users
and https://dev.twitch.tv/docs/v5/reference/streams
First you'll need to Register an application
From that you'll need to get your Client-ID.
The one in this example is not a real
TWITCH_STREAM_API_ENDPOINT_V5 = "https://api.twitch.tv/kraken/streams/{}"
API_HEADERS = {
'Client-ID' : 'tqanfnani3tygk9a9esl8conhnaz6wj',
'Accept' : 'application/vnd.twitchtv.v5+json',
}
reqSession = requests.Session()
def checkUser(userID): #returns true if online, false if not
url = TWITCH_STREAM_API_ENDPOINT_V5.format(userID)
try:
req = reqSession.get(url, headers=API_HEADERS)
jsondata = req.json()
if 'stream' in jsondata:
if jsondata['stream'] is not None: #stream is online
return True
else:
return False
except Exception as e:
print("Error checking user: ", e)
return False
I hated having to go through the process of making an api key and all those things just to check if a channel was live, so i tried to find a workaround:
As of june 2021 if you send a http get request to a url like https://www.twitch.tv/CHANNEL_NAME, in the response there will be a "isLiveBroadcast": true if the stream is live, and if the stream is not live, there will be nothing like that.
So i wrote this code as an example in nodejs:
const fetch = require('node-fetch');
const channelName = '39daph';
async function main(){
let a = await fetch(`https://www.twitch.tv/${channelName}`);
if( (await a.text()).includes('isLiveBroadcast') )
console.log(`${channelName} is live`);
else
console.log(`${channelName} is not live`);
}
main();
here is also an example in python:
import requests
channelName = '39daph'
contents = requests.get('https://www.twitch.tv/' +channelName).content.decode('utf-8')
if 'isLiveBroadcast' in contents:
print(channelName + ' is live')
else:
print(channelName + ' is not live')
It looks like Twitch provides an API (documentation here) that provides a way to get that info. A very simple example of getting the feed would be:
import urllib2
url = 'http://api.justin.tv/api/stream/list.json?channel=FollowGrubby'
contents = urllib2.urlopen(url)
print contents.read()
This will dump all of the info, which you can then parse with a JSON library (XML looks to be available too). Looks like the value returns empty if the stream isn't live (haven't tested this much at all, nor have I read anything :) ). Hope this helps!
RocketDonkey's fine answer seems to be outdated by now, so I'm posting an updated answer for people like me who stumble across this SO-question with google.
You can check the status of the user EXAMPLEUSER by parsing
https://api.twitch.tv/kraken/streams/EXAMPLEUSER
The entry "stream":null will tell you that the user if offline, if that user exists.
Here is a small Python script which you can use on the commandline that will print 0 for user online, 1 for user offline and 2 for user not found.
#!/usr/bin/env python3
# checks whether a twitch.tv userstream is live
import argparse
from urllib.request import urlopen
from urllib.error import URLError
import json
def parse_args():
""" parses commandline, returns args namespace object """
desc = ('Check online status of twitch.tv user.\n'
'Exit prints are 0: online, 1: offline, 2: not found, 3: error.')
parser = argparse.ArgumentParser(description = desc,
formatter_class = argparse.RawTextHelpFormatter)
parser.add_argument('USER', nargs = 1, help = 'twitch.tv username')
args = parser.parse_args()
return args
def check_user(user):
""" returns 0: online, 1: offline, 2: not found, 3: error """
url = 'https://api.twitch.tv/kraken/streams/' + user
try:
info = json.loads(urlopen(url, timeout = 15).read().decode('utf-8'))
if info['stream'] == None:
status = 1
else:
status = 0
except URLError as e:
if e.reason == 'Not Found' or e.reason == 'Unprocessable Entity':
status = 2
else:
status = 3
return status
# main
try:
user = parse_args().USER[0]
print(check_user(user))
except KeyboardInterrupt:
pass
Here is a more up to date answer using the latest version of the Twitch API (helix). (kraken is deprecated and you shouldn't use GQL since it's not documented for third party use).
It works but you should store the token and reuse the token rather than generate a new token every time you run the script.
import requests
client_id = ''
client_secret = ''
streamer_name = ''
body = {
'client_id': client_id,
'client_secret': client_secret,
"grant_type": 'client_credentials'
}
r = requests.post('https://id.twitch.tv/oauth2/token', body)
#data output
keys = r.json();
print(keys)
headers = {
'Client-ID': client_id,
'Authorization': 'Bearer ' + keys['access_token']
}
print(headers)
stream = requests.get('https://api.twitch.tv/helix/streams?user_login=' + streamer_name, headers=headers)
stream_data = stream.json();
print(stream_data);
if len(stream_data['data']) == 1:
print(streamer_name + ' is live: ' + stream_data['data'][0]['title'] + ' playing ' + stream_data['data'][0]['game_name']);
else:
print(streamer_name + ' is not live');
📚 Explanation
Now, the Twitch API v5 is deprecated. The helix API is in place, where an OAuth Authorization Bearer AND client-id is needed. This is pretty annoying, so I went on a search for a viable workaround, and found one.
🌎 GraphQL
When inspecting Twitch's network requests, while not being logged in, I found out the anonymous API relies on GraphQL. GraphQL is a query language for APIs.
query {
user(login: "USERNAME") {
stream {
id
}
}
}
In the graphql query above, we are querying a user by their login name. If they are streaming, the stream's id will be given. If not, None will be returned.
🐍 The Final Code
The finished python code, in a function, is below. The client-id is taken from Twitch's website. Twitch uses the client-id to fetch information for anonymous users. It will always work, without the need of getting your own client-id.
import requests
# ...
def checkIfUserIsStreaming(username):
url = "https://gql.twitch.tv/gql"
query = "query {\n user(login: \""+username+"\") {\n stream {\n id\n }\n }\n}"
return True if requests.request("POST", url, json={"query": query, "variables": {}}, headers={"client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko"}).json()["data"]["user"]["stream"] else False
I've created a website where you can play with Twitch's GraphQL API. Refer to the GraphQL Docs for help on GraphQL syntax! There's also Twitch GraphQL API documentation on my playground.
Use the twitch api with your client_id as a parameter, then parse the json:
https://api.twitch.tv/kraken/streams/massansc?client_id=XXXXXXX
Twitch Client Id is explained here: https://dev.twitch.tv/docs#client-id,
you need to register a developer application: https://www.twitch.tv/kraken/oauth2/clients/new
Example:
import requests
import json
def is_live_stream(streamer_name, client_id):
twitch_api_stream_url = "https://api.twitch.tv/kraken/streams/" \
+ streamer_name + "?client_id=" + client_id
streamer_html = requests.get(twitch_api_stream_url)
streamer = json.loads(streamer_html.content)
return streamer["stream"] is not None
I'll try to shoot my shot, just in case someone still needs an answer to this, so here it goes
import requests
import time
from twitchAPI.twitch import Twitch
client_id = ""
client_secret = ""
twitch = Twitch(client_id, client_secret)
twitch.authenticate_app([])
TWITCH_STREAM_API_ENDPOINT_V5 = "https://api.twitch.tv/kraken/streams/{}"
API_HEADERS = {
'Client-ID' : client_id,
'Accept' : 'application/vnd.twitchtv.v5+json',
}
def checkUser(user): #returns true if online, false if not
userid = twitch.get_users(logins=[user])['data'][0]['id']
url = TWITCH_STREAM_API_ENDPOINT_V5.format(userid)
try:
req = requests.Session().get(url, headers=API_HEADERS)
jsondata = req.json()
if 'stream' in jsondata:
if jsondata['stream'] is not None:
return True
else:
return False
except Exception as e:
print("Error checking user: ", e)
return False
print(checkUser('michaelreeves'))
https://dev.twitch.tv/docs/api/reference#get-streams
import requests
# ================================================================
# your twitch client id
client_id = ''
# your twitch secret
client_secret = ''
# twitch username you want to check if it is streaming online
twitch_user = ''
# ================================================================
#getting auth token
url = 'https://id.twitch.tv/oauth2/token'
params = {
'client_id':client_id,
'client_secret':client_secret,
'grant_type':'client_credentials'}
req = requests.post(url=url,params=params)
token = req.json()['access_token']
print(f'{token=}')
# ================================================================
#getting user data (user id for example)
url = f'https://api.twitch.tv/helix/users?login={twitch_user}'
headers = {
'Authorization':f'Bearer {token}',
'Client-Id':f'{client_id}'}
req = requests.get(url=url,headers=headers)
userdata = req.json()
userid = userdata['data'][0]['id']
print(f'{userid=}')
# ================================================================
#getting stream info (by user id for example)
url = f'https://api.twitch.tv/helix/streams?user_id={userid}'
headers = {
'Authorization':f'Bearer {token}',
'Client-Id':f'{client_id}'}
req = requests.get(url=url,headers=headers)
streaminfo = req.json()
print(f'{streaminfo=}')
# ================================================================
This solution doesn't require registering an application
import requests
HEADERS = { 'client-id' : 'kimne78kx3ncx6brgo4mv6wki5h1ko' }
GQL_QUERY = """
query($login: String) {
user(login: $login) {
stream {
id
}
}
}
"""
def isLive(username):
QUERY = {
'query': GQL_QUERY,
'variables': {
'login': username
}
}
response = requests.post('https://gql.twitch.tv/gql',
json=QUERY, headers=HEADERS)
dict_response = response.json()
return True if dict_response['data']['user']['stream'] is not None else False
if __name__ == '__main__':
USERS = ['forsen', 'offineandy', 'dyrus']
for user in USERS:
IS_LIVE = isLive(user)
print(f'User {user} live: {IS_LIVE}')
Yes.
You can use Twitch API call https://api.twitch.tv/kraken/streams/YOUR_CHANNEL_NAME and parse result to check if it's live.
The below function returns a streamID if the channel is live, else returns -1.
import urllib2, json, sys
TwitchChannel = 'A_Channel_Name'
def IsTwitchLive(): # return the stream Id is streaming else returns -1
url = str('https://api.twitch.tv/kraken/streams/'+TwitchChannel)
streamID = -1
respose = urllib2.urlopen(url)
html = respose.read()
data = json.loads(html)
try:
streamID = data['stream']['_id']
except:
streamID = -1
return int(streamID)