Develop Reference

How to pass variable from script 1 to script 2 in python

I need help please.
I have 2 scripts. The first script consumes from RabbitMQ and I need to send the body received to a variable in script 2.
However, the variable remains empty. I think that script 1 maybe is calling script 2 before the value is received from RabbitMQ?
How can I achieve this? Thanks
script 1
import pika
import time
from script2 import strQueue
class ReceiveFromMQ(object):
def __init__(self):
credentials = pika.PlainCredentials('xxxx', 'xxxx')
parameters = pika.ConnectionParameters('xxxx', xxx, 'xxx',
credentials)
self.connection = pika.BlockingConnection(parameters)
self.channel = self.connection.channel()
self.channel.basic_qos(prefetch_count=1)
self.channel.basic_consume(
queue='queue',
on_message_callback=self.on_response,
auto_ack=True)
self.response = None
self.channel.start_consuming()
def on_response(self, ch, method, props, body):
self.response = body.decode()
strQueue = body.decode()
print(" [x] Received %r" % body.decode())
# getMsg(body.decode())
time.sleep(body.count(b'.'))
print(" [x] Done")
print(' [*] Waiting for messages. To exit press CTRL+C')
return self.response
def call(self):
self.response = None
self.connection.process_data_events(time_limit=None)
print(str(self.response))
return str(self.response)
receive_mq = ReceiveFromMQ()
response = receive_mq.call()
print(response)
script 2
import requests
import json
strQueue = None
# Function Authenticate
def httpAuthenticate (in_apiusers, in_apipass, in_Tenant, in_URL):
try:
print('retrieve token...')
url = in_URL
payload = json.dumps({
"password": str(in_apipass),
"usernameOrEmailAddress": str(in_apiusers),
"tenancyName": str(in_Tenant)
})
headers = {
'Content-Type': 'application/json'
}
response = requests.request("POST", url, headers=headers, data=payload)
json_object = json.loads(response.text)
print('token code: ' + str(response.status_code))
return str(json_object["result"])
except Exception as e:
return 'Fail:'
# Function:Add Queue Item on Uipath Orchestrator
def httpAddQueueItems(in_URL, in_Token, in_QueueName, in_strjson):
try:
print('add queue item...')
url = in_URL
payload = json.dumps({
"itemData": {
"Priority": "Normal",
"Name": str(in_QueueName),
"SpecificContent": {
"in_pjsorequest": in_strpjson
},
"Reference": "ggg"
}
})
headers = {
'X-UIPATH-OrganizationUnitId': '',
'Content-Type': 'application/json',
'Authorization': 'Bearer ' + in_Token
}
response = requests.request("POST", url, headers=headers, data=payload)
except Exception as e:
print(e)
return 'Fail'
# CONSTANTS
OnPremuser = "xxxx"
OnPrempass = "xxx!"
OnPremtenant = "Default"
OnPremUrlAuth = "xxxx"
OnPremUrlAddQueue = "https://xxxx"
OnPremQueue = "JSON"
OnPremPJSON = strQueue
OnPremtoken = httpAuthenticate(OnPremuser, OnPrempass, OnPremtenant, OnPremUrlAuth)
httpAddQueueItems(OnPremUrlAddQueue, OnPremtoken, OnPremQueue, OnPremJSON)

What you are trying to achieve is not possible in this way since you are
trying to access a shared variable (Race Condition).
Moreover, only one bytecode instruction can be executed at a time, mean to
say, only one CPU bound task can be run at a time.
P.S:- It can be achieved by running a consumer for the RabbitMQ producer and then assign the json received to a variable.

How to modify the content-length post modification of response in a middleware in FastAPI?

Example Code:
# Here is a minimal reproducible example
import json
from starlette.datastructures import MutableHeaders
from starlette.types import ASGIApp, Receive, Scope, Send, Message
import datetime
import socket
import uvicorn
from fastapi import FastAPI
class MetaDataAdderMiddleware:
application_generic_urls = ['/openapi.json', '/docs', '/docs/oauth2-redirect', '/redoc']
def __init__(self, app: ASGIApp) -> None:
self.app = app
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
start_time = datetime.datetime.utcnow()
async def send_wrapper(message: Message) -> None:
if message["type"] == "http.response.body" and len(message["body"]) and not any([scope["path"].startswith(endpoint) for endpoint in MetaDataAdderMiddleware.application_generic_urls]):
response_body = json.loads(message["body"].decode())
end_time = datetime.datetime.utcnow()
response_processing_time_seconds = end_time - start_time
data = {}
data["data"] = response_body
data['metadata'] = {
'request_timestamp_utc': start_time,
'response_timestamp_utc': end_time,
'processing_time_seconds': response_processing_time_seconds,
'service_host': socket.gethostname()
}
data_to_be_sent_to_user = json.dumps(data, default=str).encode("utf-8")
message["body"] = data_to_be_sent_to_user
await send(message)
await self.app(scope, receive, send_wrapper)
app = FastAPI(
title="MY DUMMY APP",
)
app.add_middleware(MetaDataAdderMiddleware)
#app.get("/")
async def root():
return {"message": "Hello World"}
Description:
So here is my usecase: All of my endpoints in FastAPI APP, whatever response they are sending, I need to wrap that response, with some metadata. Let's say, some endpoint is sending me this: {"data_key": "data_value"}. But, the users should see, this as the final output:
{
"data": {"data_key": "data_value"}
"metadata": {
"request_timestamp_utc": "somevalue",
...and so on
}
}
I have a big application, and numerous routers. We have achieved the functionality of adding Request ID, Authentication and Authorization, so far by writing middlewares.
However, when I hit APIs of my app, after adding the abovementioned MetaDataAdderMiddleware, I am greeted with this following error:
ERROR: Exception in ASGI application
Traceback (most recent call last):
File "<MY PYTHON PATH>/lib/python3.6/site-packages/uvicorn/protocols/http/httptools_impl.py", line 521, in send
raise RuntimeError("Response content longer than Content-Length")
RuntimeError: Response content longer than Content-Length
This error is logical enough, since I have modified the Response body but not changed the content-length headers.
Here is snapshot of message and scope bodies in the send_wrapper function level, and as well as header values I have printed from the httptools_impl.py level:
(I have edited out some fields, to mask org-specific things)
send_wrapper called
message: {'type': 'http.response.start', 'status': 200, 'headers': [(b'content-length', b'58'), (b'content-type', b'application/json')]}
scope: {'type': 'http', 'asgi': {'version': '3.0', 'spec_version': '2.1'}, 'http_version': '1.1', 'scheme': 'http', 'method': 'POST', 'root_path': '', 'query_string': b'', 'headers': [(b'content-type', b'application/json'), (b'accept', b'*/*'), (b'cache-control', b'no-cache'), (b'accept-encoding', b'gzip, deflate'), (b'content-length', b'238'), (b'connection', b'keep-alive')], 'app': <fastapi.applications.FastAPI object at >, 'fastapi_astack': <contextlib2.AsyncExitStack object at >, 'router': <fastapi.routing.APIRouter object at >, 'endpoint': <function initiate_playbook_execution at >, 'path_params': {}, 'route': <fastapi.routing.APIRoute object at >}
INFO: - "POST /MYAPI" 200 OK
INSIDE httptools_impl
name: b'content-length' | value: b'58'
self.expected_content_length: 58
send_wrapper called
message: {'type': 'http.response.body', 'body': b'{"status":true,"stdout":null,"stderr":null,"message":null}'}
scope: {'type': 'http', 'asgi': {'version': '3.0', 'spec_version': '2.1'}, 'http_version': '1.1', 'scheme': 'http', 'method': 'POST', 'root_path': '', 'query_string': b'', 'headers': [(b'content-type', b'application/json'), (b'accept', b'*/*'), (b'cache-control', b'no-cache'), (b'accept-encoding', b'gzip, deflate'), (b'content-length', b'238'), (b'connection', b'keep-alive')], 'app': <fastapi.applications.FastAPI object at >, 'fastapi_astack': <contextlib2.AsyncExitStack object at >, 'router': <fastapi.routing.APIRouter object at >, 'endpoint': <function initiate_playbook_execution at >, 'path_params': {}, 'route': <fastapi.routing.APIRoute object at >}
INSIDE httptools_impl
body: b'{"data": {"status": true, "stdout": null, "stderr": null, "message": null}, "metadata": {"request_timestamp_utc": "BLAH", "response_timestamp_utc": "BLAH", "processing_time_seconds": "0:00:00.469472", "some_field": "some_value"}}'
num_bytes: 286
Here are the attempts that I have made to update the content-length:
In the send wrapper function just after I update the response body, I have tried doing the following:
data_to_be_sent_to_user = json.dumps(data, default=str).encode("utf-8")
message["body"] = data_to_be_sent_to_user
headers = MutableHeaders(scope=scope)
headers["content-length"] = str(len(data_to_be_sent_to_user))
# But this hasn't worked, no change in situation!
How can I proceed forward?

Thanks to #MatsLindh comment, I referred to Starlette's GZipMiddleware codebase here: https://github.com/encode/starlette/blob/fcc4c705ff69182ebd663bc686cb55c242d32683/starlette/middleware/gzip.py#L60
So the idea is, the problematic content-length value is in header present in http.response.start message. So, how GZipMiddleware has been written is, they have simply not sent this first http.response.start message instantly. Instead, they also capture http.response.body, then modify the response, then find its length, then update the length in http.response.start message, and then send both these messages in the correct order.
The working implementation that I was able to write, borrowing heavily from GZipMiddleware is here:
from starlette.types import ASGIApp, Receive, Scope, Send, Message
from starlette.requests import Request
import json
from starlette.datastructures import MutableHeaders
class MetaDataAdderMiddleware:
application_generic_urls = ['/openapi.json', '/docs', '/docs/oauth2-redirect', '/redoc']
def __init__(
self,
app: ASGIApp
) -> None:
self.app = app
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] == "http" and not any([scope["path"].startswith(endpoint) for endpoint in MetaDataAdderMiddleware.application_generic_urls]):
responder = MetaDataAdderMiddlewareResponder(self.app, self.standard_meta_data, self.additional_custom_information)
await responder(scope, receive, send)
return
await self.app(scope, receive, send)
class MetaDataAdderMiddlewareResponder:
def __init__(
self,
app: ASGIApp,
) -> None:
"""
"""
self.app = app
self.initial_message: Message = {}
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
self.send = send
await self.app(scope, receive, self.send_with_meta_response)
async def send_with_meta_response(self, message: Message):
message_type = message["type"]
if message_type == "http.response.start":
# Don't send the initial message until we've determined how to
# modify the outgoing headers correctly.
self.initial_message = message
elif message_type == "http.response.body":
response_body = json.loads(message["body"].decode())
data = {}
data["data"] = response_body
data['metadata'] = {
'field_1': 'value_1',
'field_2': 'value_2'
}
data_to_be_sent_to_user = json.dumps(data, default=str).encode("utf-8")
headers = MutableHeaders(raw=self.initial_message["headers"])
headers["Content-Length"] = str(len(data_to_be_sent_to_user))
message["body"] = data_to_be_sent_to_user
await self.send(self.initial_message)
await self.send(message)

Why does Cloud Tasks gives "Deadline Exceeded" error after 60s?

I'm using google-cloud-tasks==2.2.0 with Flask Gunicorn. This is how I send a task to a queue:
def send_task(payload, queue, uri, *args):
url = f'https://www.mywebsite.com/{uri}'
payload = json.dumps(payload)
payload = payload.encode()
parent = client.queue_path(project=project, location=location, queue=queue)
service_account_email = 'myaccount.com'
# Construct the request body.
td = '1800s'
duration = duration_pb2.Duration()
time = duration.FromJsonString(td)
now = datetime.utcnow() + timedelta(seconds=10)
ts = timestamp_pb2.Timestamp()
now = ts.FromDatetime(now)
task = {
'http_request': { # Specify the type of request.
'http_method': tasks_v2beta3.HttpMethod.POST,
'url': url,
'body': payload, # Convert dictionary to string
'headers': { # Add custom header
'Content-Type': 'application/json'
},
'oidc_token': {'service_account_email': service_account_email}
}
}
task['schedule_time'] = now
task['dispatch_deadline'] = time
response = client.create_task(request={"parent": parent, "task": task}, timeout=30.0)
I use dispatch_deadline which is supposed to support 30 minutes timeout, using this API reference.
But no matter how I try, my task fails after 60 seconds with 504 DEADLINE_EXCEEDED error.
Honestly, is this something necessary I'm missing here, or is it a bug?

How do I make an API call and authenticate it with a given API key using Python?

This is my code to extract player data from an endpoint containing basketball data for a Data Science project.NOTE: I changed the name of the actual API key I was given since it's subscription. And I change the username/password because for privacy purposes. Using the correct credentials, I wouldn't receive a syntax error but the status code always returns 401. Since it wasn't accepting the API key, I added my account username, password, and the HTTP authentication header as well, but the status code still returns 401.
In case this is relevant, this is the website's recommendation in the developer portal: **The API key can be passed either as a query parameter or using the following HTTP request header.
Please let me know what changes I can make to my code. Any help is appreciated.
Ocp-Apim-Subscription-Key: {key}**
PS: My code got fragmented while posting this, but it is all in one function.
def getData():
user_name = "name#gmail.com"
api_endpoint = "https://api.sportsdata.io/v3/nba/stats/json/PlayerGameStatsByDate/2020-FEB7"
api_key = "a45;lkf"
password = "ksaljd"
header = "Ocp-Apim-Subscription-Key"
PARAMS = {'user': user_name, 'pass': password, 'header': header, 'key': api_key}
response = requests.get(url = api_endpoint, data = PARAMS)
print(response.status_code)
file = open("Data.csv", "w")
file.write(response.text)
file.close()

def _get_auth_headers() -> dict:
return {
'Content-Type': 'application/json',
'Ocp-Apim-Subscription-Key': "`Insert key here`"
}
api_endpoint = "https://api.sportsdata.io/v3/nba/stats/json/PlayerGameStatsByDate/2020-FEB7"
PARAMS = {
# Your params here
}
response = requests.get(
api_endpoint,
headers=_get_auth_headers(),
params=PARAMS
)

Instead of just a string, you need to pass dict in the headers parameter and auth param exist so you can use it as follow:
def getData():
[...]
header = {
"Ocp-Apim-Subscription-Key": api_key
}
[...]
response = requests.get(url = api_endpoint, data = PARAMS, headers=header, auth = (user_name, password))

According to the API documentation you don't need to provide email and password. You're only need to add your API Key to header:
import requests
r = requests.get(url='https://api.sportsdata.io/v3/nba/stats/json/PlayerGameStatsByDate/2020-FEB7', headers={'Ocp-Apim-Subscription-Key': 'API_KEY'})
print(r.json())
Output:
[{
'StatID': 768904,
'TeamID': 25,
'PlayerID': 20000788,
'SeasonType': 1,
'Season': 2020,
'Name': 'Tim Hardaway Jr.',
'Team': 'DAL',
'Position': 'SF',
'Started': 1,
'FanDuelSalary': 7183,
'DraftKingsSalary': 7623,
'FantasyDataSalary': 7623,
...

Response class in Flask-RESTplus

What is the proper way to handle response classes in Flask-RESTplus?
I am experimenting with a simple GET request seen below:
i_throughput = api.model('Throughput', {
'date': fields.String,
'value': fields.String
})
i_server = api.model('Server', {
'sessionId': fields.String,
'throughput': fields.Nested(i_throughput)
})
#api.route('/servers')
class Server(Resource):
#api.marshal_with(i_server)
def get(self):
servers = mongo.db.servers.find()
data = []
for x in servers:
data.append(x)
return data
I want to return my data in as part of a response object that looks like this:
{
status: // some boolean value
message: // some custom response message
error: // if there is an error store it here
trace: // if there is some stack trace dump throw it in here
data: // what was retrieved from DB
}
I am new to Python in general and new to Flask/Flask-RESTplus. There is a lot of tutorials out there and information. One of my biggest problems is that I'm not sure what to exactly search for to get the information I need. Also how does this work with marshalling? If anyone can post good documentation or examples of excellent API's, it would be greatly appreciated.

https://blog.miguelgrinberg.com/post/customizing-the-flask-response-class
from flask import Flask, Response, jsonify
app = Flask(__name__)
class CustomResponse(Response):
#classmethod
def force_type(cls, rv, environ=None):
if isinstance(rv, dict):
rv = jsonify(rv)
return super(MyResponse, cls).force_type(rv, environ)
app.response_class = CustomResponse
#app.route('/hello', methods=['GET', 'POST'])
def hello():
return {'status': 200, 'message': 'custom_message',
'error': 'error_message', 'trace': 'trace_message',
'data': 'input_data'}
result
import requests
response = requests.get('http://localhost:5000/hello')
print(response.text)
{
"data": "input_data",
"error": "error_message",
"message": "custom_message",
"status": 200,
"trace": "trace_message"
}