Develop Reference

Passing data from http api to bigquery using google cloud function python

I am new to google cloud function and python but I managed to check online and write the below code in the main.py file but unable to get the data into bigquery
import pandas as pd
import json
import requests
from pandas.io import gbq
import pandas_gbq
import gcsfs
def validate_http(request):
request_json = request.get_json()
if request.args:
get_api_data()
return f'Data pull complete'
elif request_json:
get_api_data()
return f'Data pull complete'
else:
get_api_data()
return f'Data pull complete'
def get_api_data():
import requests
import pprint
headers = { 'Content-Type': 'application/x-www-form-urlencoded', }
data = f'client_id={my_client_id}&client_secret={my_client_secret}&grant_type=client_credentials&scope={my_scope}'
response = requests.post('https://login.microsoftonline.com/4fa9c138-d3e7-4bc3-8bab-a74bde6b7584/oauth2/v2.0/token', headers=headers, data=data)
json_response = response.json()
access_token = json_response["access_token"]
import requests
from requests.structures import CaseInsensitiveDict
url = "my_api_url"
headers = CaseInsensitiveDict()
headers["Accept"] = "application/json"
headers["Authorization"] = f"Bearer {access_token}"
resp = requests.get(url, headers=headers)
import json
new_json_response = resp.json()
new_json_response2 = new_json_response["value"]
j_data = json.dumps(new_json_response2)
input_data = j_data
data = json.loads(input_data)
result = [json.dumps(record) for record in data]
ndjson = "\n".join(result)
df = pd.DataFrame.from_records(ndjson)
bq_load('TABLE_NAME', df)
def bq_load(key, value):
project_name = 'PROJECT_ID'
dataset_name = 'DATASET_NAME'
table_name = key
value.to_gbq(destination_table='{}.{}'.format(dataset_name, table_name), project_id=project_name, if_exists='replace')
Can anyone help with what the issue is or if there is another way to get my json data to bigquery with python and google cloud function?
I have created a table in bigquery with the dataframe as well as per the screenshot below
panda_dataframe
Error message is below
error message in logs

Python API conversion from Python2.7 code to Python3

I am trying to convert this part into Python3 and just can't get it to work.
import urllib
import datetime
import urllib2
import httplib
import hmac
from hashlib import sha256
TIMEOUT=60
def getDimMetrics(options):
now = datetime.datetime.now()
# create a dictionary of the arguments to the API call
post_dict = {}
if options.group:
post_dict['group'] = options.group
# start and end are dates, the format will be specified on the command line
# we will simply pass those along in the API
if options.start:
post_dict['start'] = options.start
if options.end:
post_dict['end'] = options.end
# encode the data
post_data = urllib.urlencode(post_dict)
protocol = 'https'
if 'localhost' in options.host:
# for testing
protocol = 'http'
url = protocol + '://' + options.host + options.url + '.' + options.type.lower()
# create the request
request = urllib2.Request(url, post_data)
# add a date header (optional)
request.add_header('Date', str(now))
# calculate the authorization header and add it
hashString = 'POST' + request.get_selector() + request.get_header('Date') + 'application/x-www-form-urlencoded' + str(len(request.get_data()))
calc_sig = hmac.new(str(options.secret), hashString,
sha256).hexdigest()
request.add_header('Authorization', 'Dimo %s:%s' %(options.key, calc_sig))
print 'url=', url
print 'post_data=', post_data
print 'headers=', request.headers
This is what I have in Python3. When I run it I get an error message saying 400 Malformed Authorization header. How can I fix this error so that I can get this part running in python 3.
from requests_oauthlib import OAuth1Session
CONSUMER_KEY = "aaaaaaaaaaaaaaaaa"
CONSUMER_SECRET = "bbbbbbbbbbbbbbbbbbbb"
host = "api.dimo.com"
uri = "/api/Visits.json"
oauthRequest = OAuth1Session(CONSUMER_KEY,
client_secret=CONSUMER_SECRET)
url = 'https://' + host + uri
headers = {
'Accept': "application/json",
'Accept-Encoding': "application/x-www-form-urlencoded",
'Authorization': "dimo bbbbbbbbbbbbbbb"
}
response = oauthRequest.post(url, headers=headers)
print(response.status_code)
print(response.content)
ERROR
400
b'Malformed Authorization header.\n'

Problem
You're experiencing issues converting python 2.7 code to python 3.x.
Your authorization header in your original python 3 converted code isn't valid.
Solution
Run python 2to3 converter with some additional cleanup based on PEP 8 -- Style Guide for Python ( max line length, etc.).
Example
import datetime
import urllib.request
import urllib.error
import urllib.parse
import http.client
import hmac
from hashlib import sha256
TIMEOUT = 60
def getDimMetrics(options):
now = datetime.datetime.now()
# create a dictionary of the arguments to the API call
post_dict = {}
if options.group:
post_dict['group'] = options.group
# start and end are dates, the format will be specified on the command line
# we will simply pass those along in the API
if options.start:
post_dict['start'] = options.start
if options.end:
post_dict['end'] = options.end
# encode the data
post_data = urllib.parse.urlencode(post_dict)
protocol = 'https'
if 'localhost' in options.host:
# for testing
protocol = 'http'
url = f'{protocol}://{options.host}{options.url}.' \
f'{options.type.lower()}'
# create the request
request = urllib.request.Request(url, post_data)
# add a date header (optional)
request.add_header('Date', str(now))
# calculate the authorization header and add it
hashString = f'POST{request.get_selector()}' \
f'{request.get_header('Date')}' \
f'application/x-www-form-urlencoded' \
f'{str(len(request.get_data()))}'
calc_sig = hmac.new(
str(options.secret), hashString, sha256).hexdigest()
request.add_header('Authorization', 'Dimo {options.key}:{calc_sig}')
print(f'url={url}')
print(f'post_data={post_data}')
print(f'headers={request.headers}')
References
Python 2to3: https://docs.python.org/3/library/2to3.html

immobilienscout24 perform Two-legged OAuth Python

i have been trying to perform Two-legged OAuth at : https://rest.immobilienscout24.de/restapi/api/ . I am following these instructions but i am receiving 401 error code.
Here is my code :
import requests
URL = "https://rest.immobilienscout24.de/restapi/api/"
consumer_key = 'ImmobilieAnalyseKey'
PARAMS = {'consumer_key': consumer_key}
r = requests.get(url=URL, params=PARAMS)
print(r)
I have tried this and it is returning 200 but i am not getting how can i get data with two legged.
from rauth import OAuth1Session
consumer_key = 'xxxxx'
consumer_secret = 'xxxxx'
base_url = 'https://rest.immobilienscout24.de/restapi/api/'
def test_get():
url = base_url
client = OAuth1Session(consumer_key,consumer_secret)
r = client.get(url)
print(r)
test_get()
EDIT :
ERROR RESPONSE :
<common:messages xmlns:common="http://rest.immobilienscout24.de/schema/common/1.0">
<message>
<messageCode>ERROR_BAD_REQUEST</messageCode>
<message>Missing signature method.</message>
</message>
</common:messages>

Python Webhook: Passing through a URL + payload

I'm a beginner with Python and trying to build a service that takes information from api.ai, passes it to an API, then returns a confirmation message from the JSON it returns.
app.py:
#!/usr/bin/env python
from __future__ import print_function
from future.standard_library import install_aliases
install_aliases()
from urllib.parse import urlparse, urlencode
from urllib.request import urlopen, Request
from urllib.error import HTTPError
import json
import os
import sys
import logging
from flask import Flask, render_template
from flask import request
from flask import make_response
# Flask app should start in global layout
app = Flask(__name__)
app.logger.addHandler(logging.StreamHandler(sys.stdout))
app.logger.setLevel(logging.ERROR)
#app.route('/webhook', methods=['POST'])
def webhook():
req = request.get_json(silent=True, force=True)
print("Request:")
print(json.dumps(req, indent=4))
res = processRequest(req)
res = json.dumps(res, indent=4)
# print(res)
r = make_response(res)
r.headers['Content-Type'] = 'application/json'
return r
def processRequest(req):
if req.get("result").get("action") != "bookMyConference":
return {}
#oauth
orequest = req.get("originalRequest") # work down the tree
odata = orequest.get("data") # work down the tree
user = odata.get("user") # work down the tree
access_token = user.get("access_token")
#data
result = req.get("result") # work down the tree
parameters = result.get("parameters") # work down the tree
startdate = parameters.get("start-date")
meetingname = parameters.get("meeting-name")
payload = {
"start-date": startdate,
"end-date": startdate,
"meeting-name": meetingname
}
# POST info to join.me
baseurl = "https://api.join.me/v1/meetings"
p = Request(baseurl)
p.add_header('Content-Type', 'application/json; charset=utf-8')
p.add_header('Authorization', 'Bearer ' + access_token) #from oauth
jsondata = json.dumps(payload)
jsondataasbytes = jsondata.encode('utf-8') # needs to be bytes
jresult = urlopen(p, jsondataasbytes).read()
data = json.loads(jresult)
res = makeWebhookResult(data)
return res
def makeWebhookResult(data):
speech = "Appointment scheduled!"
print("Response:")
print(speech)
return {
"speech": speech,
"displayText": speech,
# "data": data,
"source": "heroku-bookmyconference"
}
if __name__ == '__main__':
port = int(os.getenv('PORT', 5000))
print("Starting app on port %d" % port)
app.run(debug=False, port=port, host='0.0.0.0')
Edit 4: Here's the error I'm getting in my Heroku logs:
2017-03-21T19:06:09.383612+00:00 app[web.1]: HTTPError: HTTP Error
400: Bad Request

Borrowing from here, using urlib modules inside processRequest() you could add your payload to urlopen like this:
req = Request(yql_url)
req.add_header('Content-Type', 'application/json; charset=utf-8')
jsondata = json.dumps(payload)
jsondataasbytes = jsondata.encode('utf-8') # needs to be bytes
result = urlopen(req, jsondataasbytes).read()
data = json.loads(result)
Things get more succinct if using the requests module:
headers = {'content-type': 'application/json'}
result = requests.post(yql_url, data=json.dumps(payload), headers=headers)
data = result.json()
EDIT: Adding some details specific to the join.me api
Looking at the join.me docs you'll need to obtain an access token to add to your header. But you also need an app auth code before you can get an access token. You can get the app auth code manually, or by chaining some redirects.
To get started, try this url in your browser and get the code from the callback params. Using your join.me creds:
auth_url = 'https://secure.join.me/api/public/v1/auth/oauth2' \
+ '?client_id=' + client_id \
+ '&scope=scheduler%20start_meeting' \
+ '&redirect_uri=' + callback_url \
+ '&state=ABCD' \
+ '&response_type=code'
print(auth_url) # try in browser
To get an access token:
token_url = 'https://secure.join.me/api/public/v1/auth/token'
headers = {'content-type': 'application/json'}
token_params = {
'client_id': client_id,
'client_secret': client_secret,
'code': auth_code,
'redirect_uri': callback_url,
'grant_type': 'authorization_code'
}
result = requests.post(token_url, data=json.dumps(token_params), headers=headers)
access_token = result.json().get('access_token')
Then your header for the post to /meetings would need to look like:
headers = {
'content-type': 'application/json',
'Authorization': 'Bearer ' + access_token
}

How do I perform oauth2 for Sabre Dev Network using python?

I am trying to get an authentication token from the Sabre Dev Studio. I am following the generic directions given here https://developer.sabre.com/docs/read/rest_basics/authentication(need to login to view) but I cannot figure out how to obtain a token using Python - specifically using the python-oauth2 library as it seems to be recommended to simplify the process.
Here's a sample of my code:
config = ConfigParser.ConfigParser()
config.read("conf.ini")
clientID = config.get('DEV', 'Key').strip()
clientSecret = config.get('DEV', 'SharedSecret').strip()
consumer = oauth.Consumer(key=base64.b64encode(clientID),
secret=base64.b64encode(clientSecret))
# Request token URL for Sabre.
request_token_url = "https://api.sabre.com/v1/auth/token"
# Create our client.
client = oauth.Client(consumer)
# create headers as per Sabre Dev Guidelines https://developer.sabre.com/docs/read/rest_basics/authentication
headers = {'Content-Type':'application/x-www-form-urlencoded'}
params = {'grant_type':'client_credentials'}
# The OAuth Client request works just like httplib2 for the most part.
resp, content = client.request(request_token_url, "POST", headers=headers)
print resp
print content
The response is a type 401. Where the credentials are incomplete or misformed.
Any suggestions appreciated.

I could not with oauth2, but I did with requests package I think you can get it from here
My code is:
import requests
import base64
import json
def encodeBase64(stringToEncode):
retorno = ""
retorno = base64.b64encode(stringToEncode)
return retorno
parameters = {"user": "YOUR_USER", "group": "YOUR_GROUP", "domain": "YOUR_DOMAIN", "password": "YOUR_PASSWORD"}
endpoint = "https://api.test.sabre.com/v1"
urlByService = "/auth/token?="
url = endpoint + urlByService
user = parameters["user"]
group = parameters["group"]
domain = parameters["domain"]
password = parameters["password"]
encodedUserInfo = encodeBase64("V1:" + user + ":" + group + ":" + domain)
encodedPassword = encodeBase64(password)
encodedSecurityInfo = encodeBase64(encodedUserInfo + ":" + encodedPassword)
data = {'grant_type':'client_credentials'}
headers = {'content-type': 'application/x-www-form-urlencoded ','Authorization': 'Basic ' + encodedSecurityInfo}
response = requests.post(url, headers=headers,data=data)
print "Post Request to: " + url
print response
print "Response Message: " + response.text
Regards,

First get your credentials:
Register in https://developer.sabre.com/member/register
Sign in into https://developer.sabre.com
Go to https://developer.sabre.com/apps/mykeys and get your credentials. They should look like this (where spam and eggs will look like garbage):
client_id = 'V1:spam:DEVCENTER:EXT'
client_secret = 'eggs'
Then call the /v2/auth/token endpoint in order to get an access token:
import requests
credentials = ":".join([part.encode('base64').strip()
for part in (client_id, client_secret)]
).encode('base64').strip()
url = 'https://api.test.sabre.com/v2/auth/token'
headers = {'Authorization': 'Basic ' + credentials}
params = {'grant_type': 'client_credentials'}
r = requests.post(url, headers=headers, data=params)
assert r.status_code is 200, 'Oops...'
token = r.json()
print(token)
You should get something like this:
{u'access_token': u'T1RLAQJwPBoAuz...x8zEJg**',
u'token_type': u'bearer',
u'expires_in': 604800}
Now you can call others endpoints using an Authorization header containing your access token. For example, if you want the list of supported countries:
headers = {'Authorization': 'Bearer ' + token[u'access_token']}
endpoint = 'https://api.test.sabre.com/v1/lists/supported/countries'
r = requests.get(endpoint, headers=headers)
assert r.status_code is 200, 'Oops...'
print (r.json())
If everything went well, you should receive the list of supported countries:
{u'DestinationCountries': [
{u'CountryName': u'Antigua And Barbuda', u'CountryCode': u'AG'},
{u'CountryName': u'Argentina', u'CountryCode': u'AR'},
{u'CountryName': u'Armenia', u'CountryCode': u'AM'},
{u'CountryName': u'Aruba', u'CountryCode': u'AW'},
...
}

I'm pretty sure you're using this oauth library: https://pypi.python.org/pypi/oauth2 Despite being named "oauth2" it does not implement the OAuth 2 specification. Here is the best library I could find that provides OAuth 2 support and has documentation: http://requests-oauthlib.readthedocs.org/en/latest/oauth2_workflow.html