Passing data from http api to bigquery using google cloud function python - python

I am new to google cloud function and python but I managed to check online and write the below code in the main.py file but unable to get the data into bigquery
import pandas as pd
import json
import requests
from pandas.io import gbq
import pandas_gbq
import gcsfs
def validate_http(request):
request_json = request.get_json()
if request.args:
get_api_data()
return f'Data pull complete'
elif request_json:
get_api_data()
return f'Data pull complete'
else:
get_api_data()
return f'Data pull complete'
def get_api_data():
import requests
import pprint
headers = { 'Content-Type': 'application/x-www-form-urlencoded', }
data = f'client_id={my_client_id}&client_secret={my_client_secret}&grant_type=client_credentials&scope={my_scope}'
response = requests.post('https://login.microsoftonline.com/4fa9c138-d3e7-4bc3-8bab-a74bde6b7584/oauth2/v2.0/token', headers=headers, data=data)
json_response = response.json()
access_token = json_response["access_token"]
import requests
from requests.structures import CaseInsensitiveDict
url = "my_api_url"
headers = CaseInsensitiveDict()
headers["Accept"] = "application/json"
headers["Authorization"] = f"Bearer {access_token}"
resp = requests.get(url, headers=headers)
import json
new_json_response = resp.json()
new_json_response2 = new_json_response["value"]
j_data = json.dumps(new_json_response2)
input_data = j_data
data = json.loads(input_data)
result = [json.dumps(record) for record in data]
ndjson = "\n".join(result)
df = pd.DataFrame.from_records(ndjson)
bq_load('TABLE_NAME', df)
def bq_load(key, value):
project_name = 'PROJECT_ID'
dataset_name = 'DATASET_NAME'
table_name = key
value.to_gbq(destination_table='{}.{}'.format(dataset_name, table_name), project_id=project_name, if_exists='replace')
Can anyone help with what the issue is or if there is another way to get my json data to bigquery with python and google cloud function?
I have created a table in bigquery with the dataframe as well as per the screenshot below
panda_dataframe
Error message is below
error message in logs

Related

Connect API to Sheets using Python

I have managed to read from the sheets separately and I have managed to write to them, on the other hand I have access to the API, I'm just trying to put it all together and by the way learn Python, could someone help me?
I can't get the API response to be written in the sheet, it doesn't give me any error, it only tells me that it has connected but that it hasn't updated any column or any cell
from __future__ import print_function
import os.path
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from google.oauth2 import service_account
import requests
import json
#-------------------------------------------------------------------
def temilla ():
BASE_URL = "https://company.com"
token = 'RYE_'
#headers = {'Content-Type':'application/x-www-form-urlencoded','Authorization': "Bearer {}".format(token)}
PARAMS = {"employee_ids":"1007",
#"employee_ids":"1388",
#"business_unit_ids":"13",
"start_date":"08/07/2022",
"end_date":"09/07/2022"
}
headers = {"Content-Type": "application/json",'Authorization': "Bearer {}".format(token), "Api-version": "1.0"}
response = requests.get(BASE_URL, params = PARAMS, headers = headers )
result = [response.content]
print(response.content)
#--------------------------------------------------------------------------
SERVICE_ACCOUNT_FILE = 'keys.json'
SCOPES = ['https://www.googleapis.com/auth/spreadsheets']
creds = None
creds = service_account.Credentials.from_service_account_file(
SERVICE_ACCOUNT_FILE, scopes=SCOPES)
#--------------------------------------------------------------------------
SCOPES = ['https://www.googleapis.com/auth/spreadsheets.readonly']
SAMPLE_SPREADSHEET_ID = '146456464646464'
service = build('sheets', 'v4', credentials=creds)
sheet = service.spreadsheets()
result = sheet.values().get(spreadsheetId=SAMPLE_SPREADSHEET_ID,
range="Horarios!A1:J20").execute()
print(result)
values = result.get('values', [])
temilla()
Cosilla = temilla()
#Cosilla = [["DIA",1],["AÑO",2022],["MES",9],["EMPLEADO","MARIANO"],["ACTIVO",True]]
request = sheet.values().update(spreadsheetId=SAMPLE_SPREADSHEET_ID,
range="Prueba-Escritura!a1:xa", valueInputOption="USER_ENTERED", body=
{"values":Cosilla}).execute()
#print(result)
#print(values)
print(request)
The response that the API gives me is similar to the following
b'[{"shift_id":2400298,"business_unit_id":10,"business_unit":"alguno","employee_id":1007,"employee_code":"11111111","entry":"2022-08-07T11:30:00","exit":"2022-08-07T15:30:00"},{"shift_id":2400299,"business_unit_id":10,"business_unit":"alguno","employee_id":1007,"employee_code":"1111111","entry":"2022-08-07T19:00:00","exit":"2022-08-07T23:00:00"}]'
I believe your goal is as follows.
You want to put the values from response = requests.get(BASE_URL, params = PARAMS, headers = headers ) to Google Spreadsheet.
Sample value of it is [{"shift_id":2400298,"business_unit_id":10,"business_unit":"alguno","employee_id":1007,"employee_code":"11111111","entry":"2022-08-07T11:30:00","exit":"2022-08-07T15:30:00"},{"shift_id":2400299,"business_unit_id":10,"business_unit":"alguno","employee_id":1007,"employee_code":"1111111","entry":"2022-08-07T19:00:00","exit":"2022-08-07T23:00:00"}].
You want to achieve this using googleapis for python.
You have already been able to put and get values to Spreadsheet using Sheets API.
In this case, how about the following modification?
Modified script:
# Retrieve values from requests as JSON.
response = requests.get(BASE_URL, params = PARAMS, headers = headers ) # This is from your script.
obj = response.json()
# Convert JSON to 2 dimensional array.
keys = obj[0].keys()
ar = []
for o in obj:
temp = []
for k in keys:
temp.append(o[k] if k in o else "")
ar.append(temp)
# Put values to Spreadsheet using googleapis.
SAMPLE_SPREADSHEET_ID = "###" # <--- Please set your Spreadsheet.
request = service.spreadsheets().values().update(spreadsheetId=SAMPLE_SPREADSHEET_ID, range="Prueba-Escritura!a1", valueInputOption="USER_ENTERED", body={"values": [list(keys), *ar]}).execute()
print(request)
In this modification,
Values from response = requests.get(BASE_URL, params = PARAMS, headers = headers ) is converted to JSON. And, using this object, an array for putting into Spreadsheet is created.
Values are put from cell "A1" of the sheet of "Prueba-Escritura". If you want to change this, please modify range="Prueba-Escritura!a1".
The header values are keys = obj[0].keys(). If you want to change this order, please modify this. If you want to use the specific order, please put the header row here like keys = ['shift_id','business_unit_id','business_unit',,,].
This script puts the values including the header row. If you don't want to include the header row, please modify [list(keys), *ar] to ar.
Reference:
Method: spreadsheets.values.update

Kraken Futures API - authenticationError Python

I'm finding the Kraken Futures API confusing compared to other providers. Using a demo account I'm trying to make basic private requests and not working so far with authentication error. The code mainly comes from Kraken docs (non-futures)
Futures auth doc: https://support.kraken.com/hc/en-us/articles/360022635592-Generate-authentication-strings-REST-API-
api_sec = "MxA2FwIQxCxsfy2XDa4R8PwTjwLKjzT8GSOw+qOVuWGh3Lx6PtyW0f94J5XXKz9mP8bztRJSDQJVKBsHFicrDr/N"
api_url = "https://futures.kraken.com/derivatives/api/v3"
api_key = 'Y7kVv/hW0JWRRAhJtA8BuJkUX+E0gWmTL5NWf4lRPN8f+iYoJp9AoYwW'
def get_kraken_signature(urlpath, data, secret):
postdata = urllib.parse.urlencode(data)
encoded = (str(data['nonce']) + postdata).encode()
message = urlpath.encode() + hashlib.sha256(encoded).digest()
mac = hmac.new(base64.b64decode(secret), message, hashlib.sha256)
sigdigest = base64.b64encode(mac.digest())
return sigdigest.decode()
# Attaches auth headers and returns results of a get request
def kraken_request(uri_path, data, api_key, api_sec):
headers = {}
headers['API-Key'] = api_key
# get_kraken_signature() as defined in the 'Authentication' section
headers['API-Sign'] = get_kraken_signature(uri_path, data, api_sec)
req = requests.get((api_url + uri_path), headers=headers, data=data)
return req
# Construct the request and print the result
resp = kraken_request('/accounts', {
"nonce": str(int(1000*time.time()))
}, api_key, api_sec)
Output
{"result":"error","error":"authenticationError","serverTime":"2022-05-13T10:14:50.838Z"}

Python Webhook: Passing through a URL + payload

I'm a beginner with Python and trying to build a service that takes information from api.ai, passes it to an API, then returns a confirmation message from the JSON it returns.
app.py:
#!/usr/bin/env python
from __future__ import print_function
from future.standard_library import install_aliases
install_aliases()
from urllib.parse import urlparse, urlencode
from urllib.request import urlopen, Request
from urllib.error import HTTPError
import json
import os
import sys
import logging
from flask import Flask, render_template
from flask import request
from flask import make_response
# Flask app should start in global layout
app = Flask(__name__)
app.logger.addHandler(logging.StreamHandler(sys.stdout))
app.logger.setLevel(logging.ERROR)
#app.route('/webhook', methods=['POST'])
def webhook():
req = request.get_json(silent=True, force=True)
print("Request:")
print(json.dumps(req, indent=4))
res = processRequest(req)
res = json.dumps(res, indent=4)
# print(res)
r = make_response(res)
r.headers['Content-Type'] = 'application/json'
return r
def processRequest(req):
if req.get("result").get("action") != "bookMyConference":
return {}
#oauth
orequest = req.get("originalRequest") # work down the tree
odata = orequest.get("data") # work down the tree
user = odata.get("user") # work down the tree
access_token = user.get("access_token")
#data
result = req.get("result") # work down the tree
parameters = result.get("parameters") # work down the tree
startdate = parameters.get("start-date")
meetingname = parameters.get("meeting-name")
payload = {
"start-date": startdate,
"end-date": startdate,
"meeting-name": meetingname
}
# POST info to join.me
baseurl = "https://api.join.me/v1/meetings"
p = Request(baseurl)
p.add_header('Content-Type', 'application/json; charset=utf-8')
p.add_header('Authorization', 'Bearer ' + access_token) #from oauth
jsondata = json.dumps(payload)
jsondataasbytes = jsondata.encode('utf-8') # needs to be bytes
jresult = urlopen(p, jsondataasbytes).read()
data = json.loads(jresult)
res = makeWebhookResult(data)
return res
def makeWebhookResult(data):
speech = "Appointment scheduled!"
print("Response:")
print(speech)
return {
"speech": speech,
"displayText": speech,
# "data": data,
"source": "heroku-bookmyconference"
}
if __name__ == '__main__':
port = int(os.getenv('PORT', 5000))
print("Starting app on port %d" % port)
app.run(debug=False, port=port, host='0.0.0.0')
Edit 4: Here's the error I'm getting in my Heroku logs:
2017-03-21T19:06:09.383612+00:00 app[web.1]: HTTPError: HTTP Error
400: Bad Request
Borrowing from here, using urlib modules inside processRequest() you could add your payload to urlopen like this:
req = Request(yql_url)
req.add_header('Content-Type', 'application/json; charset=utf-8')
jsondata = json.dumps(payload)
jsondataasbytes = jsondata.encode('utf-8') # needs to be bytes
result = urlopen(req, jsondataasbytes).read()
data = json.loads(result)
Things get more succinct if using the requests module:
headers = {'content-type': 'application/json'}
result = requests.post(yql_url, data=json.dumps(payload), headers=headers)
data = result.json()
EDIT: Adding some details specific to the join.me api
Looking at the join.me docs you'll need to obtain an access token to add to your header. But you also need an app auth code before you can get an access token. You can get the app auth code manually, or by chaining some redirects.
To get started, try this url in your browser and get the code from the callback params. Using your join.me creds:
auth_url = 'https://secure.join.me/api/public/v1/auth/oauth2' \
+ '?client_id=' + client_id \
+ '&scope=scheduler%20start_meeting' \
+ '&redirect_uri=' + callback_url \
+ '&state=ABCD' \
+ '&response_type=code'
print(auth_url) # try in browser
To get an access token:
token_url = 'https://secure.join.me/api/public/v1/auth/token'
headers = {'content-type': 'application/json'}
token_params = {
'client_id': client_id,
'client_secret': client_secret,
'code': auth_code,
'redirect_uri': callback_url,
'grant_type': 'authorization_code'
}
result = requests.post(token_url, data=json.dumps(token_params), headers=headers)
access_token = result.json().get('access_token')
Then your header for the post to /meetings would need to look like:
headers = {
'content-type': 'application/json',
'Authorization': 'Bearer ' + access_token
}

Using Oauth library in python

I am running a twitter api in python using oauth library. I have included the code below. When I run the code "twtest.py", I get the error `'module' object has no attribute 'OAuthConsumer'.
1.twtest.py
import urllib
from twurl import augment
print '* Calling Twitter...'
url = augment('https://api.twitter.com/1.1/statuses/user_timeline.json',
{'screen_name': 'saurabhpathak20', 'count': '2'} )
print url
connection = urllib.urlopen(url)
data = connection.read()
print data
headers = connection.info().dict
print headers
2.twurl.py
import urllib
import oauth
import hidden
def augment(url, parameters) :
secrets = hidden.oauth()
consumer = oauth.OAuthConsumer(secrets['consumer_key'], secrets['consumer_secret'])
token = oauth.OAuthToken(secrets['token_key'],secrets['token_secret'])
oauth_request = oauth.OAuthRequest.from_consumer_and_token(consumer,
token=token, http_method='GET', http_url=url, parameters=parameters)
oauth_request.sign_request(oauth.OAuthSignatureMethod_HMAC_SHA1(), consumer, token)
return oauth_request.to_url()
def test_me() :
print '* Calling Twitter...'
url = augment('https://api.twitter.com/1.1/statuses/user_timeline.json',
{'screen_name': 'saurabhpathak20', 'count': '2'} )
print url
connection = urllib.urlopen(url)
data = connection.read()
print data
headers = connection.info().dict
print headers
3.hidden.py
def oauth() :
return { "consumer_key" : "pj......U8fFRyjV",
"consumer_secret" : "zty3njhO4IRl........ELh1YC1j1rX",
"token_key" : "515167047-xaRfSm7.......wBBOrjNd61anI55D",
"token_secret" : " y7ZCBDf6d..........x1eJV8mHRnL8hh" }
Kindly help me to understand what is wrong in the code.
Thanks.
You need to import oauth from oauth instead of import oauth
>>> from oauth import oauth
>>> oauth.OAuthConsumer
<class 'oauth.oauth.OAuthConsumer'>

Creating a python wrapper for an API using classes

I am trying to make a Python Wrapper for an API. I have been able to create scripts that work fine but don't use classes. I want to make a real wrapper of that API using classes. I am new to OOP in Python.
Following was my attempt but I am stuck at how to convert it to an OO type.
import urllib2
from urllib import urlencode
import json
class apiclient:
def __init__(self,
request_url,
hmh_api_key,
client_id,
grant_type="password",
username="username",
password="password"):
values = {
"client_id": client_id,
"grant_type": grant_type,
"username": username,
"password": password
}
data = urlencode(values)
req = urllib2.Request(request_url, data)
req.add_header("Api-Key", api_key)
response = urllib2.urlopen(req)
response_header = response.info().dict
response_body = response.read()
json_acceptable_string = response_body.replace("'", "\"")
response_body_dict = json.loads(json_acceptable_string)
return response_body_dict ## this is the response
if __name__ == "__main__":
API_KEY = "75b5cc58a5cdc0a583f91301cefedf0c"
CLIENT_ID = "ef5f7a03-58e8-48d7-a38a-abbd2696bdb6.hmhco.com"
REQUEST_URL = "http://some.url"
client = apiclient(request_url=REQUEST_URL,
api_key=API_KEY,
client_id=CLIENT_ID)
print client
Without classes, I get the response JSON as response_body_dict but with classes I get TypeError: __init__() should return None. How should I start designing my program.
I have shown only a part of the whole program, there are a lot many similar scripts that send requests to URLs and get JSON responses.
Thanks!
You should not return something from __init__ function.
EDIT:
If you need that value you should use the response_body_dict as a class member and get him from other method:
import urllib2
from urllib import urlencode
import json
class apiclient:
def __init__(self,
request_url,
api_key,
client_id,
grant_type="password",
username="username",
password="password"):
values = {
"client_id": client_id,
"grant_type": grant_type,
"username": username,
"password": password
}
data = urlencode(values)
req = urllib2.Request(request_url, data)
req.add_header("Api-Key", api_key)
response = urllib2.urlopen(req)
response_header = response.info().dict
response_body = response.read()
json_acceptable_string = response_body.replace("'", "\"")
self.response_body_dict = json.loads(json_acceptable_string)
def get_response_body(self):
return self.response_body_dict
if __name__ == "__main__":
API_KEY = "75b5cc58a5cdc0a583f91301cefedf0c"
CLIENT_ID = "ef5f7a03-58e8-48d7-a38a-abbd2696bdb6.hmhco.com"
REQUEST_URL = "http://some.url"
client = apiclient(request_url=REQUEST_URL,
api_key=API_KEY,
client_id=CLIENT_ID)
response = client.get_response_body()
print client

Categories

Resources