I have a file called main.py as follows:
from typing import List, Dict
from fastapi import Body, FastAPI
import pandas as pd
#app.post("/")
async def read_df(data: List[Dict], id_process: int = Body(...)):
return {"id_process": id_process, "data": data}
Now, if I run the code for the test, saved in the file test_main.py, that is:
from fastapi.testclient import TestClient
from main import app
import pandas as pd
import json
df = pd.DataFrame([[0.2, 0.8], [0.9, 0.1]], columns=["c1", "c2"])
json_df = json.loads(df.to_json(orient="records"))
client = TestClient(app)
def test_read_df():
response = client.post(
"/",
json={
"id_process": 1,
"data": json_df # (1)
}
)
return response.json()
print(test_read_df())
I get, as excepted:
{'id_process': 1, 'data': [{'c1': 0.2, 'c2': 0.8}, {'c1': 0.9, 'c2': 0.1}]}
Is this the standard way to pass a dataframe in an API call? Or is there a way to directly pass the dataframe df in # (1)? My goal is always to return the dictionary above.
The Process seems to be better, There are some modules like "beneath". Yet this method has the less execution time.
I suggest to continue with this.
Related
Im trying to score user input againt a model hosted in azure databricks. For this I can only use urllib which is a standard library inside python. I did use Requests for my need and below is the code.
import numpy as np
import pandas as pd
import requests
def create_tf_serving_json(data):
return {'inputs': {name: data[name].tolist() for name in data.keys()} if isinstance(data, dict) else data.tolist()}
def score_model(model_uri, databricks_token, data):
headers = {
"Authorization": f"Bearer {databricks_token}",
"Content-Type": "application/json",
}
data_json = data.to_dict(orient='records') if isinstance(data, pd.DataFrame) else create_tf_serving_json(data)
response = requests.request(method='POST', headers=headers, url=model_uri, json=data_json)
if response.status_code != 200:
raise Exception(f"Request failed with status {response.status_code}, {response.text}")
return response.json()
# Scoring a model that accepts pandas DataFrames
data = pd.DataFrame([{
"Pclass": 1,
"Age": 22,
"SibSp": 1,
"Parch": 1,
"Fare" :50
}])
score_model("My Model URL", 'Databricks Token', data)
Can someone help me to do the same with urllib.
Thanks in advance!
I have a small web server:
# app.py
from typing import List
from fastapi import FastAPI
from pydantic import BaseModel
app = FastAPI()
class Item(BaseModel):
id: int
name: str
#app.post("/items")
async def items_list(items: List[Item]):
return items
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8000)
and a python file that posts to the endpoint:
# req.py
import asyncio
from typing import List
import datetime as dt
import aiohttp
from app import Item
async def main():
data = [
Item(id=1, name='A').dict(),
Item(id=2, name='B').dict()
]
async with aiohttp.ClientSession() as session:
async with session.post(
'http://localhost:8000/items',
json=data
) as response:
print(f'response: {await response.json()}')
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
It works, I can get an output like:
response: [{'id': 1, 'name': 'A'}, {'id': 2, 'name': 'B'}]
If I set data as (no .dict()):
data = [
Item(id=1, name='A'),
Item(id=2, name='B')
]
it doesn't work because Item is not json serializable.
My goal is to post list of items to the fastapi endpoint.
This working example doesn't work if I extend Item as this:
class Item(BaseModel):
id: int
name: str
created_at: dt.datetime
created_at is a datetime and even I em using Items(..).dict() it is not json serializable.
Funny thing is that if I create an Item as:
Item(id=1, name='A',created_at=dt.datetime.utcnow()).json()
its json is perfect:
{"id": 1, "name": "A", "created_at": "2021-12-15T21:10:36.077435"}
but as aiohttp session.post(json...) uses non pydantic json encoder, Item is not json serializable.
I tried to create a new pydantic object:
class ItemsList(BaseModel):
data: List[Item]
end set it as:
data = [
Item(id=1, name='A', created_at=dt.datetime.utcnow()),
Item(id=2, name='B', created_at=dt.datetime.utcnow())
]
data_list = ItemsList(data=data)
Again, pydantic is clever enough to produce proper json:
data_list.json()
{"data": [{"id": 1, "name": "A", "created_at": "2021-12-15T21:17:34.368555"}, {"id": 2, "name": "B", "created_at": "2021-12-15T21:17:34.368555"}]}
but I am not sure how to send such json using aiohttp post.
My question is: How to by using aiohttp post a list of pydantic objects that contain datetime property to fastapi endpoint?
I would be satisfied with sending/receiving list of itmes ([Item, Item, ... Item])
One way to do this is simply pass your Pydantic JSON string as the raw request body:
# Using the "data_list" Pydantic object in one of your examples.
async with session.post(
'http://localhost:8000/items',
# Pass the JSON string as `data`.
data=data_list.json(),
# Manually set content-type.
content_type="application/json"
) as response:
print(f'response: {await response.json()}')
That way you bypass the automatic serialization that isn't compatible with Pydantic.
I am Getting this error. I am executing code of aws lambda function using python 3.7 to know quicksight dashboard version. Thanks in advance!
errorMessage: "Unable to marshal response: Object of type datetime is not JSON serializable",
errorType : "Runtime.MarshalError"
Code-
import boto3
import time
import sys
client = boto3.client('quicksight')
def lambda_handler(event, context):
response = client.list_dashboard_versions(AwsAccountId='11111', DashboardId='2222',MaxResults=10)
return response
I quick fix could be:
import boto3
import time
import sys
import json
client = boto3.client('quicksight')
def lambda_handler(event, context):
response = client.list_dashboard_versions(AwsAccountId='11111', DashboardId='2222',MaxResults=10)
return json.dumps(response, default=str)
Looking at https://boto3.amazonaws.com/v1/documentation/api/1.14.8/reference/services/quicksight.html#QuickSight.Client.list_dashboard_versions the return looks like this -
{
'DashboardVersionSummaryList': [
{
'Arn': 'string',
'CreatedTime': datetime(2015, 1, 1),
'VersionNumber': 123,
'Status': 'CREATION_IN_PROGRESS'|'CREATION_SUCCESSFUL'|'CREATION_FAILED'|'UPDATE_IN_PROGRESS'|'UPDATE_SUCCESSFUL'|'UPDATE_FAILED',
'SourceEntityArn': 'string',
'Description': 'string'
},
],
'NextToken': 'string',
'Status': 123,
'RequestId': 'string'
}
As you can see, CreatedTime is returned as datetime. If you want to return this as a JSON, you should transform this value.
I was struggling with this today with a method that also returns a datetime.
In my example 'JoinedTimestamp': datetime(2015, 1, 1) resulting in the same Unable to marshal response.
If you don't need the CreatedTime value you might as well remove it from the response as:
for account in list_accounts_response["Accounts"]:
if "JoinedTimestamp" in account:
del account["JoinedTimestamp"]
To follow up on Joseph Lane's answer, transforming this value could be something along the lines of:
for account in list_accounts_response["Accounts"]:
if "JoinedTimestamp" in account:
account["JoinedTimestamp"] = str(account["JoinedTimestamp"])
I'm calling a SOAP WebService using Zeep, and it returns a JSON-like response with a datetime object. I want to write a micro-service using Flask and return proper JSON response. However, Flask complains that:
TypeError: Object of type datetime is not JSON serializable
from flask import Flask
from flask_restful import Resource, Api
import datetime
app = Flask(__name__)
api = Api(app)
class foo(Resource):
def get(self, x):
zeepResponse = {
'Response': {
'Number': x,
'DateTime': datetime.datetime(2020, 1, 1, 0, 0, 0),
'Other': None
}
}
return zeepResponse
api.add_resource(foo, '/post/<int:x>')
if __name__ == '__main__':
app.run(debug=True)
To test from the command line, simply run:
% curl http://localhost:5000/post/911
Would you guide me how to convert zeepResponse (and the datetime specifically) to a proper JSON serializable structure?
Calling json.dumps(zeepResponse, default=str) seems to fix my problem. From Stack Overflow 11875770
from flask import Flask
from flask_restful import Resource, Api
import datetime
import json
app = Flask(__name__)
api = Api(app)
class foo(Resource):
def get(self, x):
zeepResponse = {
'Response': {
'Number': x,
'DateTime': datetime.datetime(2020, 1, 1, 0, 0, 0),
'Other': None
}
}
return json.loads(json.dumps(zeepResponse, default=str))
api.add_resource(foo, '/post/<int:x>')
if __name__ == '__main__':
app.run(debug=True)
I'm new to Python unit testing, and I want to mock calls to the boto3 3rd party library. Here's my stripped down code:
real_code.py:
import boto3
def temp_get_variable(var_name):
return boto3.client('ssm').get_parameter(Name=var_name)['Parameter']['Value']
test_real_code.py:
import unittest
from datetime import datetime
from unittest.mock import patch
import real_code
class TestRealCode(unittest.TestCase):
#patch('patching_config.boto3.client')
def test_get_variable(self, mock_boto_client):
response = {
'Parameter': {
'Name': 'MyTestParameterName',
'Type': 'String',
'Value': 'myValue',
'Version': 123,
'Selector': 'asdf',
'SourceResult': 'asdf',
'LastModifiedDate': datetime(2019, 7, 16),
'ARN': 'asdf'
}
}
mock_boto_client.get_variable.return_value = response
result_value = real_code.get_variable("MyTestParameterName")
self.assertEqual("myValue", result_value)
When I run it the test fails with
Expected :myValue
Actual :<MagicMock name='client().get_parameter().__getitem__().__getitem__()' id='2040071816528'>
What am I doing wrong? I thought by setting mock_boto_client.get_variable.return_value = response it would mock out the call and return my canned response instead. I don't understand why I am getting a MagicMock object instead of the return value I tried to set. I'd like to set up my test so that when the call to get_parameter is made with specific parameters, the mock returns the canned response I specified in the test.
There are two issues with your test code. The first is that when your mock object mock_boto_client called, it returns a new mock object. This means that the object that get_parameter() is being called on is different than the one you are attempting to set a return value on. You can have it return itself with the following:
mock_boto_client.return_value = mock_boto_client
You can also use a different mock object:
foo = MagicMock()
mock_boto_client.return_value = foo
The second issue that you have is that you are mocking the wrong method call. mock_boto_client.get_variable.return_value should be mock_boto_client.get_parameter.return_value. Here is the test updated and working:
import unittest
from datetime import datetime
from unittest.mock import patch
import real_code
class TestRealCode(unittest.TestCase):
#patch('boto3.client')
def test_get_variable(self, mock_boto_client):
response = {
'Parameter': {
'Name': 'MyTestParameterName',
'Type': 'String',
'Value': 'myValue',
'Version': 123,
'Selector': 'asdf',
'SourceResult': 'asdf',
'LastModifiedDate': datetime(2019, 7, 16),
'ARN': 'asdf'
}
}
mock_boto_client.return_value = mock_boto_client
mock_boto_client.get_parameter.return_value = response
result_value = real_code.get_variable("MyTestParameterName")
self.assertEqual("myValue", result_value)