I have created a Flask service and I'm having a trouble on parsing some JSON values which are stored in MongoDB. I know how to parse every object which has a single value expect the address parameter which has more values inside (e.g "address":[{"street":"Jardine Place","city":"Lowgap","postcode":...)
JSON file:
{"name":"Morton Fitzgerald","email":"mortonfitzgerald#ontagene.com","yearOfBirth":{"$numberInt":"1997"},"address":[{"street":"Jardine Place","city":"Lowgap","postcode":{"$numberInt":"18330"}}]}
{"name":"Dorthy Cobb","email":"dorthycobb#ontagene.com","yearOfBirth":{"$numberInt":"1994"}}
In the python script, I'm trying to find a student based on the email, which I'm giving to the body at Postman. After that, if a student with an address is found I need to return a message with every value inside the "address" except the "city". With the following code, I'm taking some results but from them, I need to print out only the "street" and "postcode".
Python function:
#app.route('/getStudentAddress', methods=['GET'])
def get_student_address():
# Request JSON data
data = None
try:
data = json.loads(request.data)
uuid = request.headers.get('authorization')
except Exception as e:
return Response("bad json content",status=500,mimetype='application/json')
if data == None:
return Response("bad request",status=500,mimetype='application/json')
if not "email" in data:
return Response("Information incomplete",status=500,mimetype="application/json")
if (is_session_valid(uuid)):
student = students.find_one({"email":data['email']})
if(student != None):
student = students.find_one({"address":student['address']})
if(student != None):
student = {'name':student["name"],'address':student["address"]}
return Response(json.dumps(student), status=200, mimetype='application/json')
else:
return Response("Not-found", status=400, mimetype='application/json')
else:
return Response("Not authenticated user", status=401, mimetype='application/json')
Results:
{"name": "Morton Fitzgerald", "address": [{"street": "Jardine Place", "city": "Lowgap", "postcode": 18330}]}
Error:
If I choose a registry like {"name":"Dorthy Cobb","email":"dorthycobb#ontagene.com","yearOfBirth":{"$numberInt":"1994"}} which doesn't have an address I'm taking an error of KeyError: 'address'
Any thoughts?
You might consider adding a "try to fetch" function that checks to see if the key exists and provides a default value:
def tryfetch( container, key, default=None ):
return container[key] if key in container else default
That way, you can write things like::
address = tryfetch( student, "address" )
if address:
addr = tryfetch(address[0], "street", "NA") + tryfetch(address[0], "postcode", "NA")
Related
I am doing a CRUD application using python and aws.
python file:
if count == 0:
response = table.put_item(
Item={
'grow_template_id':str(uuid.uuid4()),
'grow_template_title':str(grow_template_title),
'search_growth_template_title':str(grow_template_title.lower()),
'facility_id':str(facility_id),
'organization_id':str(organization_id),
'actions_to_render':actions_to_render,
'crop_id': str(crop_id),
'task_category_id':str(task_category_id),
if response['ResponseMetadata']['HTTPStatusCode'] == 200:
returnvals['status']=1
returnvals['message']=grow_template_title+" growth template created successfully"
else:
returnvals['status']=1
response = table.update_item(
Key={
'grow_template_id': str(grow_template_id)
},UpdateExpression='SET grow_template_title = :gtt,\
organization_id = :oid,\
crop_id = :ci,\
crop_type = :ct,\
task_category_id = :tcid,\
ExpressionAttributeValues={
":gtt":grow_template_title,
":oid":organization_id,
":ci":crop_id,
":ct":crop_type,
":tcid":task_category_id,
When i click on submit, i am getting a bug like this:
An error occurred (ValidationException)
when calling the UpdateItem operation:
One or more parameter values are not valid.
The AttributeValue for a key attribute cannot contain an empty string value.
Key: grow_template_id"
What mistake i am doing?
ive got an api that takes in an id
http://127.0.0.1:5000/api/v1/resources/books?id=u3qR4Ps4TbATrg97
looks like that
what im trying to do after that is add something to the end of the url, for example
http://127.0.0.1:5000/api/v1/resources/books?id=u3qR4Ps4TbATrg97uid=something
im not 100% sure if this is possible
# Create some test data for our catalog in the form of a list of dictionaries.
books = [
{'id': 'u3qR4Ps4TbATrg97',
'uid': 'what',
'title': 'A Fire Upon the Deep',
'author': 'Vernor Vinge',
'first_sentence': 'The coldsleep itself was dreamless.',
'year_published': '1992'}
]
#app.route('/api/v1/resources/books', methods=['GET'])
def api_id():
# Check if an ID was provided as part of the URL.
# If ID is provided, assign it to a variable.
# If no ID is provided, display an error in the browser.
if 'id' and 'uid' in request.args:
id = str(request.args['id'])
uid = str(request.args['uid'])
else:
return "Error: No id field provided. Please specify an id."
results = []
for book in books:
if book['id'] == id:
results.append(book)
if book['uid'] == uid:
results.append(book)
this is what i have so far, mostly copy pasted from here
thats no the whole file just the important bits i can think of
You can add two inputs inside the GET query like this
http://127.0.0.1:5000/api/v1/resources/books?id=u3qR4Ps4TbATrg97&uid=something
Just put an & in between!
Use request.args.get method to get parameters from your url. Also add & to your URL as a parameter separator.
from flask import Flask, request
app = Flask(__name__)
#app.route('/api/v1/resources/books')
def books():
id_ = request.args.get('id')
uid = request.args.get('uid')
return f'id: {id_}, uid: {uid}'
app.run()
Open http://127.0.0.1:5000/api/v1/resources/books?id=u3qR4Ps4TbATrg97&uid=something
in browser and you'll get:
id: u3qR4Ps4TbATrg97, uid: something
Multiple parameters|arguments are passed with & character. ?params1=5¶ms2=3. For your example: http://127.0.0.1:5000/api/v1/resources/books?id=u3qR4Ps4TbATrg97&uid=what. For the code, I would do:
from flask import Flask, request, jsonify, make_response
app = Flask(__name__)
# Create some test data for our catalog in the form of a list of dictionaries.
books = [
{
"id": "u3qR4Ps4TbATrg97",
"uid": "what",
"title": "A Fire Upon the Deep",
"author": "Vernor Vinge",
"first_sentence": "The coldsleep itself was dreamless.",
"year_published": "1992",
}
]
#app.route("/api/v1/resources/books", methods=["GET"])
def api_id():
# Check if an ID was provided as part of the URL.
# If ID is provided, assign it to a variable.
# If no ID is provided, display an error in the browser.
if set(["id","uid"]).intersection(set(request.args)):
id_ = str(request.args["id"])
uid = str(request.args["uid"])
else:
return make_response(
jsonify({"message": "Error: No id field provided. Please specify an id."}),
400,
)
results = []
for book in books:
if book["id"] == id_:
results.append(book)
if book["uid"] == uid:
results.append(book)
response = make_response(
jsonify({"message": results}),
200,
)
response.headers["Content-Type"] = "application/json"
return response
This would return status code 400 if no match and 200 when match
I am trying to retrieve stripe user's stripeSubscriptionId & stripeCustomerId. Here's my code:
#blueprint.route("/webhook", methods=["POST"]) #confirms whether a user has subscribed or not
def stripe_webhook():
payload = request.get_data(as_text=True)
sig_header = request.headers.get("Stripe-Signature")
try:
event = stripe.Webhook.construct_event(
payload, sig_header, stripe_keys["endpoint_secret"]
)
except ValueError as e:
# Invalid payload
return "Invalid payload", 400
except stripe.error.SignatureVerificationError as e:
# Invalid signature
return "Invalid signature", 400
# Handle the checkout.session.completed event
if event["type"] == "checkout.session.completed":
session = event["data"]["object"]
# Fulfill the purchase...
handle_checkout_session(session)
return "Success", 200
def handle_checkout_session(session):
subID = stripe.Customer.retrieve(id, status)
logging.warn(str(subID))
#blueprint.route("/create-checkout-session")
def create_checkout_session():
domain_url = "http://localhost:5000/"
stripe.api_key = stripe_keys["secret_key"]
try:
checkout_session = stripe.checkout.Session.create(
success_url=domain_url + "success?session_id={CHECKOUT_SESSION_ID}",
cancel_url=domain_url + "cancel",
payment_method_types=["card"],
mode="subscription",
line_items=[
{
"price": stripe_keys["price_id"],
"quantity": 1,
}
]
)
return jsonify({"sessionId": checkout_session["id"]})
except Exception as e:
return jsonify(error=str(e)), 403
Yet I am getting: stripe.error.InvalidRequestError: Could not determine which URL to request: Customer instance has invalid ID: <built-in function id>, <class 'builtin_function_or_method'>. ID should be of type str(orunicode)
I took doc as a reference. However, I can't seem to figure out how to retrieve stripeSubscriptionId & stripeCustomerId from the webhook or in any other way for the past couple of hours despite reading all the docs. I have seen other SO pages but couldn't find one that had a similar concern as mine with a workable solution.
subID = stripe.Customer.retrieve(id, status)
logging.warn(str(subID))
You're not getting the id anywhere here. You likely want to use session["id"]. Also I'm not sure what status is supposed to be there?
I'm developing a REST api with Django and REST-framework. I have endpoint which takes a POST request with this kind of json:
{
"pipeline": ["Bayes"],
"material": [
"Rakastan iloisuutta!",
"Autojen kanssa pitää olla varovainen.",
"Paska kesä taas. Kylmää ja sataa"
]
}
It is a machine-learning analysis api and the json tells to use Bayes classifier to provided strings and return results. This works fine when I test it manually by doing the post requests. However, it breaks down when I try to write an unit test. I have the following test:
class ClassifyTextAPITests(APITestCase):
fixtures = ['fixtures/analyzerfixtures.json'] #suboptimal fixture since requires bayes.pkl in /assets/classifiers folder
def test_classification(self):
""" Make sure that the API will respond correctly when required url params are supplied.
"""
response = self.client.post(reverse('analyzer_api:classifytext'), {
"pipeline": ["Bayes"],
"material": [
"Rakastan iloisuutta!",
"Autojen kanssa pitää olla varovainen.",
"Paska kesä taas. Kylmää ja sataa",
]
})
self.assertTrue(status.is_success(response.status_code))
self.assertEqual(response.data[0], 1)
test fails everytime because of the latter assert gives "AssertionError: 'P' != 1"
Here is my view code:
class ClassifyText(APIView):
"""
Takes text snippet as a parameter and returns analyzed result.
"""
authentication_classes = (authentication.TokenAuthentication,)
permission_classes = (permissions.AllowAny,)
parser_classes = (JSONParser,)
def post(self, request, format=None):
try:
self._validate_post_data(request.data)
print("juttu", request.data["material"])
#create pipeline from request
pipeline = Pipeline()
for component_name in request.data["pipeline"]:
pipeline.add_component(component_name)
response = pipeline.execute_pipeline(request.data['material'])
status_code = status.HTTP_200_OK
except Exception as e:
response = {"message": "Please provide a proper data.",
"error": str(e) }
status_code = status.HTTP_400_BAD_REQUEST
return Response(response, status=status_code)
def _validate_post_data(self, data):
if "pipeline" not in data:
raise InvalidRequest("Pipeline field is missing. Should be array of components used in analysis. Available components at /api/classifiers")
if len(data["pipeline"]) < 1:
raise InvalidRequest("Pipeline array is empty.")
if "material" not in data:
raise InvalidRequest("Material to be analyzed is missing. Please provide an array of strings.")
if len(data["material"]) < 1:
raise InvalidRequest("Material to be analyzed is missing, array is empty. Please provide an array of strings.")
The really interesting part was when I fired the debugger to check what happens here. Turns out that the line
request.data['material']
gives the last entry of the list in in my request, in this case
"Paska kesä taas. Kylmää ja sataa"
However, while I inspect the contents of the request.data, it shows a querydict with lists pipeline and material as they are in request. Why do I get string instead of material list when I call request.data["material"] ? Is there something I have forgotten and I have to specify some kind of serializer? And why it works during normal execution but not with tests?
I'm using Django 1.8 with Python 3. Also, I'm not tying the view to any specific model.
Finally here is what my debugger shows when I put break points into view:
request.data:
QueryDict: {'material': ['Rakastan iloisuutta!', 'Autojen kanssa pitää olla varovainen.', 'Paska kesä taas. Kylmää ja sataa'], 'pipeline': ['Bayes']}
asd = request.data["material"]:
'Paska kesä taas. Kylmää ja sataa'
This is because QueryDict returns the last value of a list in __getitem__:
QueryDict.getitem(key)
Returns the value for the given key. If the key has more than one value, getitem() returns the last value. Raises django.utils.datastructures.MultiValueDictKeyError if the key does not exist. (This is a subclass of Python’s standard KeyError, so you can stick to catching KeyError.)
https://docs.djangoproject.com/en/1.8/ref/request-response/#django.http.QueryDict.getitem
If you post a form, in which a key maps to a list:
d = {"a": 123, "b": [1,2,3]}
requests.post("http://127.0.0.1:6666", data=d)
this is what you get in the request body:
a=123&b=1&b=2&b=3
Since the test method post the data as a form, what you get from request.data is a QueryDict (the same as request.POST), hence you get the last value in the list when getting request.data.
To get expected behavior, post the data as JSON in the request body (as in #Vladir Parrado Cruz's answer).
By default the QueryDict will return a single item from the list when doing a getitem call (or access by square brackets, such as you do in request.data['material'])
You can instead use the getlist method to return all values for the key:
https://docs.djangoproject.com/en/1.8/ref/request-response/#django.http.QueryDict.getlist
class ClassifyText(APIView):
"""
Takes text snippet as a parameter and returns analyzed result.
"""
authentication_classes = (authentication.TokenAuthentication,)
permission_classes = (permissions.AllowAny,)
parser_classes = (JSONParser,)
def post(self, request, format=None):
try:
self._validate_post_data(request.data)
print("juttu", request.data["material"])
print("juttu", request.data.getlist("material"]))
#create pipeline from request
pipeline = Pipeline()
for component_name in request.data["pipeline"]:
pipeline.add_component(component_name)
response = pipeline.execute_pipeline(request.data.getlist('material'))
status_code = status.HTTP_200_OK
except Exception as e:
response = {"message": "Please provide a proper data.",
"error": str(e) }
status_code = status.HTTP_400_BAD_REQUEST
return Response(response, status=status_code)
def _validate_post_data(self, data):
if "pipeline" not in data:
raise InvalidRequest("Pipeline field is missing. Should be array of components used in analysis. Available components at /api/classifiers")
if len(data["pipeline"]) < 1:
raise InvalidRequest("Pipeline array is empty.")
if "material" not in data:
raise InvalidRequest("Material to be analyzed is missing. Please provide an array of strings.")
if len(data["material"]) < 1:
raise InvalidRequest("Material to be analyzed is missing, array is empty. Please provide an array of strings.")
Try to do something like that on the test:
import json
def test_classification(self):
""" Make sure that the API will respond correctly when required url params are supplied.
"""
response = self.client.post(
reverse('analyzer_api:classifytext'),
json.dumps({
"pipeline": ["Bayes"],
"material": [
"Rakastan iloisuutta!",
"Autojen kanssa pitää olla varovainen.",
"Paska kesä taas. Kylmää ja sataa",
]
}),
content_type='application/json'
)
self.assertTrue(status.is_success(response.status_code))
self.assertEqual(response.data[0], 1)
Perhaps if you send the data as json it will work.
Is there a better pattern for input validation than I'm using in this function?
https://github.com/nathancahill/clearbit-intercom/blob/133e4df0cfd1a146cedb3c749fc1b4fac85a6e1b/server.py#L71
Here's the same function without any validation. It's much more readable, it's short and to the point (9 LoC vs 53 LoC).
def webhook(clearbitkey, appid, intercomkey):
event = request.get_json()
id = event['data']['item']['id']
email = event['data']['item']['email']
person = requests.get(CLEARBIT_USER_ENDPOINT.format(email=email), auth=(clearbitkey, '')).json()
domain = person['employment']['domain']
company = requests.get(CLEARBIT_COMPANY_ENDPOINT.format(domain=domain), auth=(clearbitkey, '')).json()
note = create_note(person, company)
res = requests.post(INTERCOM_ENDPOINT,
json=dict(user=dict(id=id), body=note),
headers=dict(accept='application/json'),
auth=(appid, intercomkey))
return jsonify(note=res.json())
However, it doesn't handle any of these errors:
dict KeyError's (especially nested dicts)
HTTP errors
Invalid JSON
Unexpected responses
Is there a better pattern to follow? I looked into using a data validation library like voluptous but it seems like I'd still have the same problem of verbosity.
Your original code on github seems fine to me. It's a little over complicated, but also handle all cases of error. You can try to improve readability by abstract things.
Just for demonstration, I may write code like this:
class ValidationError(Exception):
"Raises when data validation fails"
pass
class CallExternalApiError(Exception):
"Raises when calling external api fails"
pass
def get_user_from_event(event):
"""Get user profile from event
:param dict event: request.get_json() result
:returns: A dict of user profile
"""
try:
event_type = event['data']['item']['type']
except KeyError:
raise ValidationError('Unexpected JSON format.')
if event_type != 'user':
return ValidationError('Event type is not supported.')
try:
id = event['data']['item']['id']
email = event['data']['item']['email']
except KeyError:
return ValidationError('User object missing fields.')
return {'id': id, 'email': email}
def call_json_api(request_function, api_name, *args, **kwargs):
"""An simple wrapper for sending request
:param request_function: function used for sending request
:param str api_name: name for this api call
"""
try:
res = request_function(*args, **kwargs)
except:
raise CallExternalApiError('API call failed to %s.' % api_name)
try:
return res.json()
except:
raise CallExternalApiError('Invalid response from %s.' % api_name)
#app.route('/<clearbitkey>+<appid>:<intercomkey>', methods=['POST'])
def webhook(clearbitkey, appid, intercomkey):
"""
Webhook endpoint for Intercom.io events. Uses this format for Clearbit and
Intercom.io keys:
/<clearbitkey>+<appid>:<intercomkey>
:clearbitkey: Clearbit API key.
:appid: Intercom.io app id.
:intercomkey: Intercom.io API key.
Supports User events, specifically designed for the User Created event.
Adds a note to the user with their employment and company metrics.
"""
event = request.get_json()
try:
return handle_event(event, clearbitkey, appid, intercomkey)
except (ValidationError, CallExternalApiError) as e:
# TODO: include **res_objs in response
return jsonify(error=str(e))
def handle_event(event):
"""Handle the incoming event
"""
user = get_user_from_event(event)
res_objs = dict(event=event)
person = call_json_api(
requests.get,
'Clearbit',
CLEARBIT_USER_ENDPOINT.format(email=user['email']),
auth=(clearbitkey, '')
)
res_objs['person'] = person
if 'error' in person:
raise CallExternalApiError('Error response from Clearbit.')
domain = person['employment']['domain']
company = None
if domain:
try:
company = call_json_api(
requests.get,
'Clearbit',
CLEARBIT_COMPANY_ENDPOINT.format(domain=domain),
auth=(clearbitkey, ''))
)
if 'error' in company:
company = None
except:
company = None
res_objs['company'] = company
try:
note = create_note(person, company)
except:
return jsonify(error='Failed to generate note for user.', **res_objs)
result = call_json_api(
requests.post,
'Intercom',
(INTERCOM_ENDPOINT, json=dict(user=dict(id=id), body=note),
headers=dict(accept='application/json'),
auth=(appid, intercomkey)
)
return jsonify(note=result, **res_objs)
I hope it helps.