SQLAlchemy / Flask is not disposing of connections - python

I have a Flask app with ORM SQLAlchemy, configured like this
def create_app():
app = Flask(__name__, static_folder='../client',
static_url_path='', template_folder='../client')
app.secret_key = secret_keys[2]
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://root:***#localhost:3307/project***'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config.update({
'SQLALCHEMY_POOL_SIZE': 100,
'SQLALCHEMY_POOL_TIMEOUT': 0
})
db.init_app(app)
Please note that the pool_size and timeout is set like that, so I can work, because the app will routinely create new and new connections and trigger the timeout (15+10). I'm not sure why this is happening, here are a few bits of code I am using:
#api.route('/api/get-variations', methods=['POST'])
def get_variations():
#
# Returns the appropriate variations for that product ID
#
product_id = request.json
data = {'data': [variation.to_dict() for variation in productvar.query.filter_by(ProductID=product_id).all()]}
return data
The model
class productvar(db.Model):
__tablename__ = 'product_variations'
id = db.Column('VariationID', db.Integer, primary_key = True)
ProductID = db.Column(db.Integer)
Variation = db.Column(db.String(50))
def to_dict(self):
return {
'Value': self.id,
'Variation': self.Variation
}
JS
function change_dropdown(htmlVariation, product) {
var productID = product
fetch('/api/get-variations', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(productID),
})
// get the variations for the product in json
.then((result) => { return result.json(); })
.then((data) => {
// replace the dropdown with the appropriate variations
var newVariations = data;
for (var i = 0; i < newVariations?.data.length; i++) {
var new_option = document.createElement("option");
new_option.text = newVariations?.data[i].Variation
new_option.value = newVariations?.data[i].Value
htmlVariation[0].appendChild(new_option);
}
})
}
The flow usually goes like this > you select an option from the website, it then goes via JS to the server and amends the dropdown (hence change_variation).
What am I doing wrong here? Shouldn't SQLAlchemy reuse the connections? Why is it constantly creating new connections? I'm imaginging that if there are 20 or more users on my website, going through various combinations, they will effectively timeout the server.
Should I go via normal SQLAlchemy and create and dispose of the engine?
Please mind that I already tried this but I can't seem to get it to actually kill the connections
db.session.close()
engine = db.get_engine(app=create_app())
engine.dispose()
All help wildly appreciated!

Related

How to retrieve the values from a post request function django views. DRF

I have a function that makes a queryset when data from my frontend is posted.
This viewset is using a serializer of combined models.
On my index page on the fronted I have a treeview filter and this view loads the query to
a config page. I need to access the event id's after this query is made to be passed to a separate task on the backend for polling.
However I cannot access the data in this function. How would I get retrieve this query and filter on the data? right now I can't print the queryset to see its structure.
Edit:
I can see the response payload now and the structure is as follows
[{"id":395,"event":{"id":175,"start_time":"28-10-20","sport_id":"9","event_id":"1576232279920003","event_name":"Botic Van de Zandschulp vs Benjamin Bonzi","status":"open"},"market":{"id":198,"market_id":"1576232286670003","market_name":"Moneyline","status":"open","volume":50.4951,"is_ip":"False","event":175},"bettriggers":null,"stakes":null,"userstakes":null,"order":null,"runner_id":"1576232286740003","name":"Botic Van de Zandschulp","back_odds":1.62893,"lay_odds":1.68028},{"id":396,"event":{"id":175,"start_time":"28-10-20","sport_id":"9","event_id":"1576232279920003","event_name":"Botic Van de Zandschulp vs Benjamin Bonzi","status":"open"},"market":{"id":198,"market_id":"1576232286670003","market_name":"Moneyline","status":"open","volume":50.4951,"is_ip":"False","event":175},"bettriggers":null,"stakes":null,"userstakes":null,"order":null,"runner_id":"1576232286780103","name":"Benjamin Bonzi","back_odds":2.47,"lay_odds":2.59}]
class CombinedViewSet(mixins.ListModelMixin,
mixins.DestroyModelMixin,
mixins.RetrieveModelMixin,
viewsets.GenericViewSet):
permission_classes = [IsMeOrReadOnly]
queryset = Runner.objects.all()
serializer_class = CombinedSerializer
#action(detail=False, methods=['POST','GET])
def loaditem(self, request):
keys = request.data['keys']
queryset = []
for xx in keys:
if xx['time_key'] is None:
if xx['event_key'] is None:
queryset.extend(Runner.objects.filter(event__sport_id=xx['sport_key']))
else:
queryset.extend(Runner.objects.filter(event__event_id=xx['event_key']))
else:
date_str = xx['time_key']
datetime_obj = datetime.datetime.strptime(date_str, '%d-%m-%y')
queryset.extend(Runner.objects.filter(event__start_time__date=datetime_obj, event__sport_id=xx['sport_key']))
serializer_class = CombinedSerializer(queryset, many=True)
return Response(serializer_class.data)
What my end goal is to pass in the filtered list of event_id's from load item function to a backend celery task
#shared_task(bind=True, base=QueueOnce, once={'graceful': True})
def get_events(self):
api = get_client()
sports = api.reference_data.get_sports()
logger.warning(f'GET EVENTS task has been called now')
events = api.market_data.get_selected_events(event_ids=['2323','34343','1576232279920003','etc'])
This is what my serializer looks like
class CombinedSerializer(serializers.ModelSerializer):
event = EventSerializer()
market = MarketSerializer()
bettriggers = BetTriggersSerializer()
stakes = StakesSerializer()
userstakes = UserStakesSerializer()
order = OrderStakesSerializer()
class Meta:
model = Runner
fields= '__all__'
Frontend code for posting data
load_item: function (event) {
if(this.value == null) return;
var len = this.value.length;
var req_key = [];
for (var x=0; x<len; x++) {
var str = ''+this.value[x];
var tmp_arr = str.split('#');
var sport_key = null;
var time_key = null;
var event_key = null;
switch(tmp_arr.length) {
case 1:
sport_key = tmp_arr[0];
break;
case 2:
event_key = tmp_arr[1];
break;
case 3:
time_key = tmp_arr[2];
sport_key = tmp_arr[1];
break;
}
req_key.push({sport_key:sport_key, event_key:event_key, time_key:time_key});
}
var vobj = this
this.$axios.post('/api/combined/loaditem/', {
keys : req_key
})
.then(function (response) {
vobj.$store.dispatch('load_runner', response.data)
vobj.$router.push('/configuration')
})
.catch(function (error) {
//currentObj.output = error;
console.log(error)
});
},

How to use flask script to connect mysql data base to my ios app

I have an app with which I can write data to my database but cannot read data from the database. I'd like to be able to send data from the database to my app whenever I want which I think is done by the get and post http methods in flask. Should this be done with the same script or separate scripts? If separate scripts, how should I go about writing them? If not I am not sure what to add to my swift and python files.
The code I am working with uses flask and mysql-connector to make database connections, any help modifying to read data from the database would be greatly appreciated:
app.py
from flask import Flask
from flask import make_response
from flask import request
import mysql.connector
#opens database connectionto "events" database
try:
mydb = mysql.connector.connect(host="localhost", user="root", passwd="Proteus^5", database = "events")
print("Connection OK")
except e:
print("Connection failed: ", e)
#prepare a cursor object using cursor() method
mycursor = mydb.cursor()
app = Flask(__name__)
#app.route('/', methods=['GET', 'POST'])
def register():
event_id = request.form['a']
poi = request.form['b']
address = request.form['c']
start_time = request.form['d']
end_time = request.form['e']
sql = "INSERT INTO events (event_id, poi, address, start_time, end_time) VALUES (%s, %s, %s, %s, %s)"
val = (event_id, poi, address, start_time, end_time)
try:
# Execute the SQL command
mycursor.execute(sql, val)
# Commit your changes in the database
mydb.commit()
mydb.close()
except e:
# Rollback in case there is any error
print("Error: ", e)
mydb.rollback()
return make_response("Success!", 200)
print("DB is closed")
swift file that sends data to database on a button press:
import Foundation
import SwiftUI
struct CreateEventButton: View {
#State private var isPresentedEvent = false
#State private var eventid: Int = 3
#State private var eventName: String = ""
#State private var eventDescription: String = ""
#State private var selectedStartTime = Date()
#State private var selectedEndTime = Date()
#Binding var annotationSelected: Bool
func send(_ sender: Any) {
let request = NSMutableURLRequest(url: NSURL(string: "http://localhost:5000/")! as
URL)
request.httpMethod = "POST"
let postString = "a=\(self.eventid)&b=\(self.eventName)&c=\.
(self.eventDescription)&d=\.
(self.selectedStartTime)&e=\(self.selectedEndTime)"
request.httpBody = postString.data(using: String.Encoding.utf8)
let task = URLSession.shared.dataTask(with: request as URLRequest) {
data, response, error in
if error != nil {
print("error=\(String(describing: error))")
return
}
print("response = \(String(describing: response))")
let responseString = NSString(data: data!, encoding:
String.Encoding.utf8.rawValue)
print("responseString = \(String(describing: responseString))")
}
task.resume()
self.eventName = ""
self.eventDescription = ""
self.selectedStartTime = Date()
self.selectedEndTime = Date()
}
var body: some View {
Button(action: {
self.isPresentedEvent.toggle() //trigger modal presentation
}, label: {
Text("Create Event").font(.system(size: 18))
}).padding(EdgeInsets(top: 8, leading: 6, bottom: 8, trailing: 6
.sheet(isPresented: $isPresentedEvent, content:{
VStack{
TextField("Event Name", text: self.$eventName).padding()
TextField("Event Description", text: self.$eventDescription).padding()
Form {
DatePicker("When your event starts: ", selection:
self.$selectedStartTime,
in: Date()...)
}
Form {
DatePicker("When your event ends: ", selection: self.$selectedEndTime,
in:
Date()...)
}
HStack{
Button(action: {
self.isPresentedEvent.toggle()
self.annotationSelected = false
self.eventid += 1
print("Start: \(self.selectedStartTime)")
print("End: \(self.selectedEndTime)")
self.send((Any).self)
}, label: {
Text("Create Event")
})
Button(action: {
self.isPresentedEvent.toggle()
}, label: {
Text("Cancel")
})
}
Text("Create Event Button (Non Functional)").padding()
}
} )
}
}
I do not know anything about swifts but I have worked with flask.
A simple and typical way for retrieving data from database is to define an API for
that. In this API you query the database (e.g. Select * From user Where user_id = ...) and then serialize the results (convert retrieved data from database in a form that is proper for sending to the client application. e.g. create a dictionary/json object) and send the response to the client.
an API example in flask
#app.route('.../get_user', methods=['GET'])
def get_user_api:
# query database
data = { 'username': username,
'level: level,
...
}
return data, 200

How to connect Flask API endpoints to React app

I have an API made on Flask that has several endpoints. I am trying to use data from these end points to display a chart on my front end that shows the data to the user.
The following are my exact requirements:
Implement one or more types of charts that can be used to effectively visualize data supplied from the API endpoints. Users should be able to pick different metrics to visualize and compare with others.
My Flask API:
import os
from flask import Flask, jsonify, session, request
import sqlalchemy
import time
from functools import wraps
# web app
app = Flask(__name__)
app.secret_key = 'super_secure_key_that_should_be_in_.env'
# database engine
engine = sqlalchemy.create_engine(os.getenv('SQL_URI'))
def rate_limit(**limit_kwargs):
def decorator(function):
#wraps(function)
def wrapper(*args, **kwargs):
limit = limit_kwargs['limit'] if 'limit' in limit_kwargs else 5
seconds = limit_kwargs['window'] if 'window' in limit_kwargs else 60
session_key = 'rate_limit_' + str(request.url_rule)
if session_key not in session:
session[session_key] = []
window: list = session[session_key]
if len(window) < limit:
window.append(int(time.time()))
session[session_key] = window
return function(*args, **kwargs)
if time.time() - window[0] < seconds:
return jsonify(error='Rate limit exceeded'), 429
window.pop(0)
window.append(int(time.time()))
session[session_key] = window
return function(*args, **kwargs)
return wrapper
return decorator
#app.route('/')
#rate_limit()
def index():
return 'Welcome 😎'
#app.route('/events/hourly')
#rate_limit(limit=3, window=20)
def events_hourly():
return queryHelper('''
SELECT date, hour, events
FROM public.hourly_events
ORDER BY date, hour
LIMIT 168;
''')
#app.route('/events/daily')
#rate_limit(limit=2)
def events_daily():
return queryHelper('''
SELECT date, SUM(events) AS events
FROM public.hourly_events
GROUP BY date
ORDER BY date
LIMIT 7;
''')
#app.route('/stats/hourly')
#rate_limit(limit=3, window=20)
def stats_hourly():
return queryHelper('''
SELECT date, hour, impressions, clicks, revenue
FROM public.hourly_stats
ORDER BY date, hour
LIMIT 168;
''')
#app.route('/stats/daily')
#rate_limit(limit=2)
def stats_daily():
return queryHelper('''
SELECT date,
SUM(impressions) AS impressions,
SUM(clicks) AS clicks,
SUM(revenue) AS revenue
FROM public.hourly_stats
GROUP BY date
ORDER BY date
LIMIT 7;
''')
#app.route('/poi')
#rate_limit(limit=3)
def poi():
return queryHelper('''
SELECT *
FROM public.poi;
''')
def queryHelper(query):
with engine.connect() as conn:
result = conn.execute(query).fetchall()
return jsonify([dict(row.items()) for row in result])
My React App.js:
import React, {Component} from 'react';
import logo from './logo.svg';
import './App.css';
import DailyEvents from "./Components/DailyEvents";
class App extends Component {
render() {
return (
<div className="App">
<header className= "App-header">
<img src={logo} className="App-logo" alt="logo" />
<h1 className="App-title"> Welcome to React</h1>
</header>
<DailyEvents />
</div>
);
}
}
export default App;
My Components/DailyEvents.js:
import React, { Component } from 'react';
import axios from 'axios';
class App extends Component {
state = {
dates: []
}
componentDidMount() {
axios.get('api_url_or_localhost/events/daily')
.then(res => res.json())
.then((data) => {
this.setState({ dates: data })
})
.catch(console.log)
}
}
export default App;
I keep getting a TypeError that instance.render is not a function.
What I am trying to do is use my Flask API endpoints to visualize the data on the front-end.
try using axios :
https://alligator.io/react/axios-react/
it should look like this :
axios.get('put the query address')
.then(response => {
process your response
})
Can / Should you make changes to the Flask API, or is that code part of the requirements? Because I'm pretty sure you won't be able to return those Date objects straight from the DB without serializing them first.
I suggest you use Postman (or just a browser, for get requests) to test the routes locally and basically have a look at the data first.
React can use Axios or fetch to get this data and visualize it.
import React, { Component } from 'react'
class App extends Component {
state = {
dates: []
}
componentDidMount() {
fetch('api_url_or_localhost/events/daily'))
.then(res => res.json())
.then((data) => {
this.setState({ dates: data })
})
.catch(console.log)
}
...
}
see this https://pusher.com/tutorials/consume-restful-api-react for more details

How to use Axios/Vue.js to call data from SQLite/Django backend?

Status Quo:
Whenever a user visits my web application, Axios makes a request to a third party API to fetch data and populate the frontend with that data using v-for.
Conclusion: I have one API call per website visitor.
Desired status:
Whenever a user visits my web application, Axios shall fetch the data from the SQLite database which itself is populated every XX seconds by a python request to reduce API calls.
Questions:
Now I implemented a SQLite database using Django models and views. So far so good, the API gets fetched regularly and updates the database table properly.
1.) How can I now call the data in the database using Axios? As by my research Axios somehow needs to call a view and the view will call the data from the database, is this correct?
2.) If Axios needs to call a view, do I need another view.py file that calls the database? If I would insert the needed view function into the existing view.py file it would initiate another API call, wouldn't it?
3.) And how can I implement the link to the view function to Axios? Instead of a third party API url would I just use the path to the view file?
Quotes_app/Views.py:
from django.shortcuts import render
from Quotes_app.models import ratesEUR
import json
import requests
response = requests.get("http://data.fixer.io/api/latest?access_key=XXXX&base=EUR")
rates_EUR = json.loads(response.content.decode('utf-8'))
timestamp = rates_EUR['timestamp']
base = rates_EUR['base']
date = rates_EUR['date']
rates = rates_EUR['rates']
id = 1
rates_new = ratesEUR(id=id, timestamp=timestamp, base=base, date=date, rates=rates)
rates_new.save()
def render_Quotes_app(request, template="Quotes_app/templates/Quotes_app/Quotes_app.html"):
return render(request, template)
Quotes_app/models.py:
from django.db import models
class ratesEUR(models.Model):
timestamp = models.CharField(max_length=10)
base = models.CharField(max_length=3)
date = models.DateField(auto_now=False, auto_now_add=False)
rates = models.CharField(max_length=8)
def __str__(self):
return self.base
Vue.js/axios: (as of now directly fetching the API)
Vue.config.devtools = true;
var app = new Vue({
delimiters: ['[[', ']]'],
el: '.eurQuotesWrapper',
data() {
return {
rates: [],
};
},
computed: {
rates1() {
const ratesArr1 = Object.entries(this.rates);
const ret = ratesArr1.reduce((a, c, i, d) => {
if (i < d.length / 2) a[c[0]] = c[1];
return a;
}, {});
console.log('rates1', ret);
return ret;
},
rates2() {
const ratesArr2 = Object.entries(this.rates);
const ret = ratesArr2.reduce((a, c, i, d) => {
if (i >= d.length / 2) a[c[0]] = c[1];
return a;
}, {});
console.log('rates2', ret);
return ret;
}
},
created() {
axios
.get("http://data.fixer.io/api/latest?access_key=XXXX&base=EUR")
.then(response => {
this.rates = response.data.rates;
console.log(this.rates);
for(key in this.rates) {
this.rates[key] = new Intl.NumberFormat('de-DE', {
minimumFractionDigits: 5,
maximumFractionDigits: 5
}).format(this.rates[key]);
}
console.log(this.rates);
});
}
});
In advance thank you very much for your help!
You can use DRF(Django Rest Framework) to make REST API's or You can use JsonResponse
to send JSON objects as response.
from django.http import JsonResponse
data = # Your data
JsonResponse(data, encoder=MyJSONEncoder)
For more details visit django's documentation
Better would be if you use DRF for making rest apis. It's much easier.

How do I render a cached JSONP view in Django

I'm querying a db of election info and spitting them out in JSONP to use for some ajax/jQuery widgets we are creating.
What I'm wanting to do however is cache the view so that a new request isn't coming in every time the widget is rendered.
I'm running into 2 issues though.
Which method of caching should I use to render this view so that the database isn't getting hit hard.
I need to render the view as JSONP which means I need the callback variable to be dynamic (which it current'y is with my script but I'm afraid it wouldn't be with a cached view).
Here's what my code looks like.
from models import race
from models import candidates
from django.http import HttpResponse
from django.utils import simplejson
def data(request):
data = []
races = race.objects.all()
for election in races:
race_candidate = candidates.objects.filter(race__id=election.pk)
candidate_info = []
for n,candidate in enumerate(race_candidate):
candidate_values = {
"name":candidate.name,
"percent":candidate.percent,
"totalvotes":candidate.totalvotes,
"partyname":candidate.partyname,
"partyabbrv":candidate.partyabbrv,
}
candidate_info.append(candidate_values)
race_values = {
"title":election.title,
"description":election.description,
"priority":election.priority,
"decided":election.decided,
"candidates":candidate_info,
}
data.append(race_values)
json_races = '{"races":' + simplejson.dumps(data) + '}'
if("callback" in request.GET.keys()):
callback = request.GET["callback"]
else:
callback = None
if(callback):
response = HttpResponse("%s(%s)" % (
callback,
simplejson.dumps(data)
), mimetype="application/json"
)
else:
response = HttpResponse(json_races, mimetype="application/json")
return response
You'll have to use the low-level cache API with whichever cache backend you like.
from django.core.cache import cache
...
CACHE_TIMEOUT = 3600 # choose your timeout
def data(request):
jsonstring = cache.get('elections')
if jsonstring is None:
data = [{
"title": election.title,
"description": election.description,
"priority": election.priority,
"decided": election.decided,
"candidates": [
{
"name": candidate.name,
"percent": candidate.percent,
"totalvotes": candidate.totalvotes,
"partyname": candidate.partyname,
"partyabbrv": candidate.partyabbrv,
} for candidate in election.candidates_set.all()
],
} for election in race.objects.all()]
jsonstring = simplejson.dumps(data)
cache.set('elections', jsonstring, CACHE_TIMEOUT)
callback = request.GET.get('callback')
if callback:
response = HttpResponse("%s(%s)" % (callback, jsonstring),
mimetype="application/json")
else:
response = HttpResponse('{"races":' + jsonstring + '}',
mimetype="application/json")
return response
In your settings.py, configure CACHE according to documentation. The memory backend is the simplest, memcached is probably the best.

Categories

Resources