how to send parameters to databricks notebook? - python

I triggering databricks notebook using the following code:
TOKEN = "xxxxxxxxxxxxxxxxxxxx"
headers = {"Authorization": "Bearer %s" % TOKEN}
data = {
"job_id": xx,
"notebook_task": {
"base_parameters": {
"param1":"key1",
"param2": "key2",
}
},
}
resp = requests.post(
"https://xxxxxx.cloud.databricks.com/api/2.0/jobs/run-now",
headers=headers,
data=json.dumps(data),
)
when i try to access it using dbutils.widgets.get("param1"), im getting the following error:
com.databricks.dbutils_v1.InputWidgetNotDefined: No input widget named param1
I tried using notebook_params also, resulting in the same error.


base_parameters is used only when you create a job. When you trigger it with run-now, you need to specify parameters as notebook_params object (doc), so your code should be :
data = {
"job_id": xx,
"notebook_params": {
"param1":"key1",
"param2": "key2",
},
}

Related

Amadeus Flight Availabilities in Python - Error 400

I am trying to use the POST method of Amadeus flight availabilities API on Python, but is is still giving me error 400. Any suggestions? Many thanks.
from amadeus import Client, ResponseError, Location
import requests
amadeus = Client(
client_id='My ID',
client_secret='My Secret')
try:
flights = amadeus.get('/v2/shopping/flight-offers',originLocationCode = "GIG",destinationLocationCode = "ATL",departureDate = "2023-01-31",nonStop = "true",travelClass = "BUSINESS",adults = 1)
body = flights.data[0]
print(body)
except ResponseError as error:
print(error)
try:
availability = amadeus.post('/v1/shopping/availability/flight-availabilities', body)
print(availability.result)
except ResponseError as error:
print('headers: ', error.response.request.headers)
print('body: ', error.response.request.params)

As per their API documentation, the body of the second API call needs to be in the following format:
{
"originDestinations": [
{
"id": "1",
"originLocationCode": "BOS",
"destinationLocationCode": "MAD",
"departureDateTime": {
"date": "2021-11-14",
"time": "21:15:00"
}
}
],
"travelers": [
{
"id": "1",
"travelerType": "ADULT"
}
],
"sources": [
"GDS"
]
}
Right now you are just feeding the second call the return from the first, which doesn't seem to be what they're looking for.
You might also need to feed the proper headers, something like:
headers = {"Content-Type": "application/json; charset=utf-8"}
then you would do
response = requests.post(url, headers=headers, json=body)

How to get the value of "authorization" from Json using python?

I have a Json data, where I want to get the value of "authorization" i.e "OToken
DEDC1071B77800A146B6E8D2530E0429E76520C151B40CC3325D8B
6D9242CBA3A6BFA643E7E5596FBEBAE0F46A1FB1BCD099EBC1F59D
CD82F390B6BC45FCE036F37F7F589BD687A691E1378F1FF432331C
62E7E641E857C8F8A405A4BFE2F01B1EB8F3C69817D45F5DDE9DEE
346ACABA1B7208DECA9E43CCE7AB3761553E23D9CB36A870C1819C
15C7C4B1CFE2802DFD05F651AA537AB81787.4145535F55415431" using python
{
"links":[
{
"method":"GET",
"rel":"self",
"href":"https://www.sampleurl.com/request"
},
{
"headers":{
"authorization":"OToken
DEDC1071B77800A146B6E8D2530E0429E76520C151B40CC3325D8B
6D9242CBA3A6BFA643E7E5596FBEBAE0F46A1FB1BCD099EBC1F59D
CD82F390B6BC45FCE036F37F7F589BD687A691E1378F1FF432331C
62E7E641E857C8F8A405A4BFE2F01B1EB8F3C69817D45F5DDE9DEE
346ACABA1B7208DECA9E43CCE7AB3761553E23D9CB36A870C1819C
15C7C4B1CFE2802DFD05F651AA537AB81787.4145535F55415431"
},
"valid_date":"2020-08-17T15:49:00+0530",
"method":"POST",
"rel":"redirect",
"href":"https://www.billdesk.com/pgi/MerchantPayment/",
"parameters":{
"mercid":"BDMERCID",
"bdorderid":"OAFC19XTFD8TSP"
}
}
]
}

import json
response = YOUR_JSON_STRING
data = json.loads(response)
authorization = data['headers']['authorization']

How do you handle graphql query and fragment in python?

I am trying to query using the requests library but am having trouble. I suspect it's to do with the handling of the fragment but I'm not sure.
When I run the code I get Response 400. Here is my code:
import requests
import json
query = """query GetAxieTransferHistory($axieId: ID!, $from: Int!, $size: Int!) {
axie(axieId: $axieId) {
id
transferHistory(from: $from, size: $size) {
...TransferRecords
__typename
}
__typename
}
}
fragment TransferRecords on TransferRecords {
total
results {
from
to
timestamp
txHash
withPrice
__typename
}
__typename
}"""
params = {
"axieId": "9082310",
"from": 0,
"size": 1
}
url = 'https://axieinfinity.com/graphql-server-v2/graphql'
r = requests.post(url, json={"query": query, "params": params})
print(r.status_code)
Thanks in advance!

Try changing your params key to variables
r = requests.post(url, json={"query": query, "variables": params})

How do I get an API response containing body, but no headers with Python & AWS Lambda?

I have created an API with AWS Lambda in Python. Unfortunately, the response contains the headers, and I would like it to only contain the body. The Lambda API looks like this:
import json
import boto3
def lambda_handler(event, context):
#Step1: Scan table
client = boto3.resource('dynamodb') #Access DynamoDB
table = client.Table("register-to-event") #Access table
response = table.scan()
return {
"statusCode": 200,
"headers": {"Content-Type": "application/json",
},
"body": response["Items"]
}
The problem is that the API response contains headers and body when I call it. This is the response:
{
"statusCode": 200,
"headers": {
"Content-Type": "application/json"
},
"body": [
{
"your-email": "hannes.hannesen#googlemail.com",
"your-number": "004785454548",
"your-last-name": "Hannesen",
"your-first-name": "Hannes",
"ID": 3
},
{
"your-email": "stig.stigesen#googlemail.com",
"your-number": "+4754875456",
"your-last-name": "Stigesen",
"your-first-name": "Stig",
"ID": 0
}
]
}
The goal is to call the API and return only the body which is json like this:
[
{
"your-email": "hannes.hannesen#googlemail.com",
"your-number": "004785454548",
"your-last-name": "Hannesen",
"your-first-name": "Hannes",
"ID": 3
},
{
"your-email": "stig.stigesen#googlemail.com",
"your-number": "+4754875456",
"your-last-name": "Stigesen",
"your-first-name": "Stig",
"ID": 0
}
]

The Solution was to configure the API in AWS correctly by ticking the box next to "Use Lambda Proxy integration":
Use Lambda Proxy integration
My next problem ended up being "serializing decimals" as a result of DynamoDB making the keys of the table a type (Decimal) which does not exist in Python.
The solution to this can be found here: https://learn-to-code.workshop.aws/persisting_data/dynamodb/step-3.html

How we can create dataproc cluster using http request, getting error Expected OAuth 2 access token,

I am new in Python and airflow. I want to create a dataproc cluster using an http request inside a pythonoperator task. See the below code:
def create_cluster():
API_ENDPOINT = "https://dataproc.googleapis.com/v1beta2/projects/trim
-**********/regions/us-central1-b/clusters"
data = {
"projectId": "trim-**********",
"clusterName": "cluster-1",
"config": {
"configBucket": "",
"gceClusterConfig": {
"subnetworkUri": "default",
"zoneUri": "us-central1-b"
},
"masterConfig": {
"numInstances": 1,
"machineTypeUri": "n1-standard-1",
"diskConfig": {
"bootDiskSizeGb": 500,
"numLocalSsds": 0
}
},
"workerConfig": {
"numInstances": 2,
"machineTypeUri": "n1-standard-1",
"diskConfig": {
"bootDiskSizeGb": 100,
"numLocalSsds": 0
}
}
}
}
r = requests.post(url=API_ENDPOINT, data=data)
pastebin_url = r.text
print("The pastebin URL is:%s" % pastebin_url)
But I am getting an error: Request is missing required authentication credential. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project.
What would be the solution for this error? Thanks in advance.

Categories

Resources