I am trying to upload raw JSON data on Google Cloud Platform. But I am getting this error:
TypeError: must be string or buffer, not dict
Code:
def upload_data(destination_path):
credentials = GoogleCredentials.get_application_default()
service = discovery.build('storage', 'v1', credentials=credentials)
content = {'name': 'test'}
media = http.MediaIoBaseUpload(StringIO(content), mimetype='plain/text')
req = service.objects().insert(
bucket=settings.GOOGLE_CLOUD_STORAGE_BUCKET,
body={"cacheControl": "public,max-age=31536000"},
media_body=media,
predefinedAcl='publicRead',
name=destination_path,
)
resp = req.execute()
return resp
Code Worked by changing StringIO(content) to StringIO(json.dumps(content))
in your example, content is a dict. Perhaps you want to use json?
content = json.dumps({'name': 'test'})
To answer my own comment question on how to get this to work:
To get this to work you'll need
from googleapiclient.discovery import build
from googleapiclient import http
from oauth2client.client import GoogleCredentials
import io
Also i needed to change this:
media = http.MediaIoBaseUpload(StringIO(content), mimetype='plain/text')
in to this:
media = http.MediaIoBaseUpload(io.StringIO(json.dumps(content)), mimetype='plain/text')
Note the insert of 'io' besides the json.dumps recommended by Corey Goldberg.
for predefinedAcl options you can go here:
https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
Bucket needs to be the full name of your bucket. If you come from firebase it's "[name].appspot.com"
Related
Several years ago I created a small Python program which were able to maintain my calendar using oauth2client which is now deprecated and replaced with google.auth - but I cannot find any useful documentation and my program stopped working complaining about a _module KeyError which nobody appear to have solved except by upgrading.
I cannot figure out how to replace the oauth2client with google.auth:
import datetime
import httplib2
import os
from apiclient import discovery
import oauth2client
from oauth2client import client
from oauth2client import tools
...
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
service = discovery.build('calendar', 'v3', http=http)
According to the oauth2client deprecation notes, the replacement to be used to manage Google user credentials is google-auth-oauthlib. Below a snippet working on my PC (python 3.6).
As the documentation highlights the new library does not save the credentials, that's why I am using pickle to save them. Maybe, depending on your application requirements, you want to have a more robust solution (like a database).
import os
import pickle
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
SCOPES = ['https://www.googleapis.com/auth/calendar.readonly', ]
# we check if the file to store the credentials exists
if not os.path.exists('credentials.dat'):
flow = InstalledAppFlow.from_client_secrets_file('client_id.json', SCOPES)
credentials = flow.run_local_server()
with open('credentials.dat', 'wb') as credentials_dat:
pickle.dump(credentials, credentials_dat)
else:
with open('credentials.dat', 'rb') as credentials_dat:
credentials = pickle.load(credentials_dat)
if credentials.expired:
credentials.refresh(Request())
calendar_sdk = build('calendar', 'v3', credentials=credentials)
calendars_get_params = {
'calendarId': 'primary',
}
test = calendar_sdk.calendars().get(**calendars_get_params).execute()
print(test)
I haven't robustly tested this, but it works for testing snippets with my personal account. I'm sure there are changes that could and/or should be made to it for enterprise applications, such as passing auth'd Http() instances, detecting scope changes, and so on.
You can review the full code on my GitHub repo:
requirements:
google-api-python-client
google-auth
google-auth-oauthlib
whatever deps the above pull in
I use the InstalledAppFlow class, and generally followed the instructions on Google's Python auth guide.
Code (Python 3.6)
# Google API imports
from googleapiclient.discovery import build
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
SCOPES = ['your scopes', 'here']
def get_saved_credentials(filename='creds.json'):
'''Read in any saved OAuth data/tokens
'''
fileData = {}
try:
with open(filename, 'r') as file:
fileData: dict = json.load(file)
except FileNotFoundError:
return None
if fileData and 'refresh_token' in fileData and 'client_id' in fileData and 'client_secret' in fileData:
return Credentials(**fileData)
return None
def store_creds(credentials, filename='creds.json'):
if not isinstance(credentials, Credentials):
return
fileData = {'refresh_token': credentials.refresh_token,
'token': credentials.token,
'client_id': credentials.client_id,
'client_secret': credentials.client_secret,
'token_uri': credentials.token_uri}
with open(filename, 'w') as file:
json.dump(fileData, file)
print(f'Credentials serialized to {filename}.')
def get_credentials_via_oauth(filename='client_secret.json', scopes=SCOPES, saveData=True) -> Credentials:
'''Use data in the given filename to get oauth data
'''
iaflow: InstalledAppFlow = InstalledAppFlow.from_client_secrets_file(filename, scopes)
iaflow.run_local_server()
if saveData:
store_creds(iaflow.credentials)
return iaflow.credentials
def get_service(credentials, service='sheets', version='v4'):
return build(service, version, credentials=credentials)
Usage is then:
creds = get_saved_credentials()
if not creds:
creds = get_credentials_via_oauth()
sheets = get_service(creds)
Buy use Google Analytics API (V4) I would like to upload file "Product Data"
This is the sample code from GA Documentation
from apiclient.http import MediaFileUpload
try:
media = MediaFileUpload('custom_data.csv',
mimetype='application/octet-stream',
resumable=False)
daily_upload = analytics.management().uploads().uploadData(
accountId='123456',
webPropertyId='UA-123456-1',
customDataSourceId='9876654321',
media_body=media).execute()
except TypeError, error:
# Handle errors in constructing a query.
print 'There was an error in constructing your query : %s' % error
except HttpError, error:
# Handle API errors.
print ('There was an API error : %s : %s' %
(error.resp.status, error.resp.reason))
This is What I ahve done, but I still have an issue regarding this part - analytics.management().uploads()
import argparse
from googleapiclient.discovery import build
from oauth2client.service_account import ServiceAccountCredentials
import httplib2
from oauth2client import client
from oauth2client import file
from oauth2client import tools
from googleapiclient.http import MediaFileUpload
# SET VARS
CUSTOM_DATA_SOURCE_ID='_xxxxxxx'
WEB_PROPERTY_ID='UA-xxxxx-1'
ACCOUNT_ID='xxxxxx'
CSV_IMPORT_FILE_LOCATION='test_file.csv'
CREDENTIALS_KEY_FILE_LOCATION='key.json'
def get_service(api_name, api_version, scope, key_file_location):
credentials = ServiceAccountCredentials.from_json_keyfile_name(
key_file_location, scopes=scope)
http = credentials.authorize(httplib2.Http())
# Build the service object.
service = build(api_name, api_version, http=http)
return service
def uploadCSV(service):
try:
media = MediaFileUpload(CSV_IMPORT_FILE_LOCATION,
mimetype='application/octet-stream',
resumable=False)
daily_upload = service.management().uploads().uploadData(
accountId=ACCOUNT_ID,
webPropertyId=WEB_PROPERTY_ID,
customDataSourceId=CUSTOM_DATA_SOURCE_ID,
media_body=media).execute()
except TypeError, error:
# Handle errors in constructing a query.
print 'There was an error in constructing your query : %s' % error
def main():
# Define the auth scopes to request.
scope = ['https://www.googleapis.com/auth/analytics.edit','https://www.googleapis.com/auth/analytics']
# Authenticate and construct service.
service = get_service('analytics', 'v4', scope, CREDENTIALS_KEY_FILE_LOCATION)
# Upload CSV Data
uploadCSV(service)
if __name__ == '__main__':
main()
This is an error which I have received all the time:
AttributeError: 'Resource' object has no attribute 'management'
Any suggestions??
I assume that this is because I do not have these methods (management().uploads()) but this is what example from documentation says.
The current analytics api v4 only includes the reporting side of the API. To access the management endpoint you have to use v3. Try to rewrite your code using the v3 version of the API.
Here's what v3 looks like
Here's what v4 looks like
To start change this line:
service = get_service('analytics', 'v3', scope, CREDENTIALS_KEY_FILE_LOCATION)
But it might require more rewriting than this.
For a project in which we created an app that records certain scores throughout the day, we also created some graphs in R which we saved as jpegs on the Raspberry.
We want to upload the jpg to Firebase via Python (we uploaded a variable to Firebase and it worked)
We first tried this code:
from google.cloud import storage
client = storage.Client()
bucket = client.get_bucket('teddy-aztech-ehealth.appspot.com')
graphicBlob = bucket.get_blob('graph.jpeg')
graphBlob.upload_from_filename(filename='/home/pi/graph.jpeg')
But we get a long error from the client bucket part , telling us the bucket name must start and end with a number.
We also tried this code:
import sys
import requests
import firebase_admin
from firebase_admin import credentials
from firebase_admin import storage
sys.argv = "/home/pi/graph.jpeg"
image_url = sys.argv
cred = credentials.Certificate('teddy-aztech-ehealth-firebase-adminsdk-t0iz1-61f49237f4.json')
firebase_admin.initialize_app(cred, {
'storageBucket': 'https://teddy-aztech-ehealth.appspot.com'
})
bucket = storage.bucket()
image_data = requests.get(image_url).content
blob = bucket.blob('graph.jpg')
blob.upload_from_string(
image_data,
content_type='image/jpg'
)
print(blob.public_url)
But get an error at the part with initializeapp (again, because of the bucket...)
Do we have to activate/give access from Firebase?
Your initial attempt is close to what you need.
import io
from google.cloud import storage
# Google Cloud Project ID. This can be found on the 'Overview' page at
# https://console.developers.google.com
PROJECT_ID = 'your-project-id'
CLOUD_STORAGE_BUCKET = 'your-bucket-name'
filename = "graph-filename.jpeg"
# Create unique filename to avoid name collisions in Google Cloud Storage
date = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H%M%S")
basename, extension = filename.rsplit('.', 1)
unique_filename = "{0}-{1}.{2}".format(basename, date, extension)
# Instantiate a client on behalf of the project
client = storage.Client(project=PROJECT_ID)
# Instantiate a bucket
bucket = client.bucket(CLOUD_STORAGE_BUCKET)
# Instantiate a blob
blob = bucket.blob(unique_filename)
# Upload the file
with open(filename, "rb") as fp:
blob.upload_from_file(fp)
# The public URL for this blob
url = blob.public_url
I'm trying to make an http post request with Microsoft's face api, in order to connect it with photos in my azure blob storage account. When I run the following code, I get multiple errors like handshake error, or ssl routines type errors. I appreciate any help! The problem code is :
api_response = requests.post(url, headers=headers, data=blob)
obviously for context here is what I ran before that. This first chunk sets up the storage account:
%matplotlib inline
import matplotlib.pyplot as plt
import io
from io import StringIO
import numpy as np
import cv2
from PIL import Image
from PIL import Image
import os
from array import array
azure_storage_account_name = 'musicsurveyphotostorage'
azure_storage_account_key = None #dont need key... we will access public blob...
if azure_storage_account_name is None:
raise Exception("You must provide a name for an Azure Storage account")
from azure.storage.blob import BlockBlobService
blob_service = BlockBlobService(azure_storage_account_name, azure_storage_account_key)
# select container (folder) name where the files resides
container_name = 'musicsurveyphotostorage'
# list files in the selected folder
generator = blob_service.list_blobs(container_name)
blob_prefix = 'https://{0}.blob.core.windows.net/{1}/{2}'
# load image file to process
blob_name = 'shiba.jpg' #name of image I have stored
blob = blob_service.get_blob_to_bytes(container_name, blob_name)
image_file_in_mem = io.BytesIO(blob.content)
img_bytes = Image.open(image_file_in_mem)
This second chunk calls out the API and the problematic post request:
#CALL OUT THE API
import requests
import urllib
url_face_api = 'https://eastus.api.cognitive.microsoft.com/face/v1.0'
api_key ='____'
#WHICH PARAMETERS ATTRIBUTES DO YOU WANT RETURNED
headers = {'Content-Type': 'application/octet-stream', 'Ocp-Apim-
Subscription-Key':api_key}
params = urllib.parse.urlencode({
'returnFaceId': 'true',
'returnFaceLandmarks': 'true',
'returnFaceAttributes': 'age,gender,smile,facialHair,headPose,glasses',
})
query_string = '?{0}'.format(params)
url = url_face_api + query_string
#THIS IS THE PROBLEM CODE
api_response = requests.post(url, headers=headers, data=blob)
#print out output in json
import json
res_json = json.loads(api_response.content.decode('utf-8'))
print(json.dumps(res_json, indent=2, sort_keys=True))
If I open the Fiddler, I also could reproduce the issue that you mentioned. If it is that case, you could pause to capture the request with fiddler during send request.
Based on my test, in your code there are 2 code lines need to be changed. From more information you could refer to the screenshot.
We also could get the some demo code from azure offical document.
url_face_api = 'https://westcentralus.api.cognitive.microsoft.com/face/v1.0/detect' # in your case miss detect
api_response = requests.post(url, headers=headers,data=blob.content) # data should be blob.content
I've created a "wrapper" class for the the python-quickstart code. It works fine in my class, unless I change the mime type. In their code they created a plain text document, but I'm trying to create a Google Docs file from my code. When I try to run this code, I receive an HttpError 400 stating that my mime type is invalid. What exactly am I doing wrong?
Here is my code:
import pprint
import httplib2
import googleapiclient.discovery
import googleapiclient.http
import googleapiclient.errors
import oauth2client.client
class DriveClient():
def __init__(self):
self.oauth2_scope = 'https://www.googleapis.com/auth/drive'
self.client_secrets = 'client_secrets.json'
self.mimetype = 'application/vnd.google-apps.document'
self.flow = self.set_flow()
self.drive_service = self.authorize_url()
def set_flow(self):
flow = oauth2client.client.flow_from_clientsecrets(self.client_secrets,
self.oauth2_scope)
flow.redirect_uri = oauth2client.client.OOB_CALLBACK_URN
return flow
def authorize_url(self):
authorize_url = self.flow.step1_get_authorize_url()
print('Go to the following link in your browser: ' + authorize_url)
code = input('Enter verification code: ').strip()
credentials = self.flow.step2_exchange(code)
http = httplib2.Http()
credentials.authorize(http)
drive_service = googleapiclient.discovery.build('drive', 'v2',
http=http)
return drive_service
def push_file(self, file_src, title, description=''):
media_body = googleapiclient.http.MediaFileUpload(
file_src, mimetype=self.mimetype, resumable=True)
body = {
'title': title,
'description': description
}
try:
new_file = self.drive_service.files().insert(body=body,
media_body=media_body
).execute()
pprint.pprint(new_file)
except googleapiclient.errors.HttpError as error:
print('An error occured: %s' % error)
if __name__ == '__main__':
d = DriveClient()
d.push_file('document.txt', 'mTitle', 'mDescription')
Try setting the mime type to the type of the source document, eg application/msword, application/vnd.oasis.opendocument.text , etc. Google needs to know what the format of the incoming document is, then it will choose what kind of google doc to create.