Develop Reference

Google Client API v3 - update a file on drive using Python

I'm trying to update the content of a file from a python script using the google client api. The problem is that I keep receiving error 403:
An error occurred: <HttpError 403 when requesting https://www.googleapis.com /upload/drive/v3/files/...?alt=json&uploadType=resumable returned "The resource body includes fields which are not directly writable.
I have tried to remove metadata fields, but didn't help.
The function to update the file is the following:
# File: utilities.py
from googleapiclient import errors
from googleapiclient.http import MediaFileUpload
from googleapiclient.discovery import build
from httplib2 import Http
from oauth2client import file, client, tools
def update_file(service, file_id, new_name, new_description, new_mime_type,
new_filename):
"""Update an existing file's metadata and content.
Args:
service: Drive API service instance.
file_id: ID of the file to update.
new_name: New name for the file.
new_description: New description for the file.
new_mime_type: New MIME type for the file.
new_filename: Filename of the new content to upload.
new_revision: Whether or not to create a new revision for this file.
Returns:
Updated file metadata if successful, None otherwise.
"""
try:
# First retrieve the file from the API.
file = service.files().get(fileId=file_id).execute()
# File's new metadata.
file['name'] = new_name
file['description'] = new_description
file['mimeType'] = new_mime_type
file['trashed'] = True
# File's new content.
media_body = MediaFileUpload(
new_filename, mimetype=new_mime_type, resumable=True)
# Send the request to the API.
updated_file = service.files().update(
fileId=file_id,
body=file,
media_body=media_body).execute()
return updated_file
except errors.HttpError as error:
print('An error occurred: %s' % error)
return None
And here there is the whole script to reproduce the problem.
The goal is to substitute a file, retrieving its id by name.
If the file does not exist yet, the script will create it by calling insert_file (this function works as expected).
The problem is update_file, posted above.
from __future__ import print_function
from utilities import *
from googleapiclient import errors
from googleapiclient.http import MediaFileUpload
from googleapiclient.discovery import build
from httplib2 import Http
from oauth2client import file, client, tools
def get_authenticated(SCOPES, credential_file='credentials.json',
token_file='token.json', service_name='drive',
api_version='v3'):
# The file token.json stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
store = file.Storage(token_file)
creds = store.get()
if not creds or creds.invalid:
flow = client.flow_from_clientsecrets(credential_file, SCOPES)
creds = tools.run_flow(flow, store)
service = build(service_name, api_version, http=creds.authorize(Http()))
return service
def retrieve_all_files(service):
"""Retrieve a list of File resources.
Args:
service: Drive API service instance.
Returns:
List of File resources.
"""
result = []
page_token = None
while True:
try:
param = {}
if page_token:
param['pageToken'] = page_token
files = service.files().list(**param).execute()
result.extend(files['files'])
page_token = files.get('nextPageToken')
if not page_token:
break
except errors.HttpError as error:
print('An error occurred: %s' % error)
break
return result
def insert_file(service, name, description, parent_id, mime_type, filename):
"""Insert new file.
Args:
service: Drive API service instance.
name: Name of the file to insert, including the extension.
description: Description of the file to insert.
parent_id: Parent folder's ID.
mime_type: MIME type of the file to insert.
filename: Filename of the file to insert.
Returns:
Inserted file metadata if successful, None otherwise.
"""
media_body = MediaFileUpload(filename, mimetype=mime_type, resumable=True)
body = {
'name': name,
'description': description,
'mimeType': mime_type
}
# Set the parent folder.
if parent_id:
body['parents'] = [{'id': parent_id}]
try:
file = service.files().create(
body=body,
media_body=media_body).execute()
# Uncomment the following line to print the File ID
# print 'File ID: %s' % file['id']
return file
except errors.HttpError as error:
print('An error occurred: %s' % error)
return None
# If modifying these scopes, delete the file token.json.
SCOPES = 'https://www.googleapis.com/auth/drive'
def main():
service = get_authenticated(SCOPES)
# Call the Drive v3 API
results = retrieve_all_files(service)
target_file_descr = 'Description of deploy.py'
target_file = 'deploy.py'
target_file_name = target_file
target_file_id = [file['id'] for file in results if file['name'] == target_file_name]
if len(target_file_id) == 0:
print('No file called %s found in root. Create it:' % target_file_name)
file_uploaded = insert_file(service, target_file_name, target_file_descr, None,
'text/x-script.phyton', target_file_name)
else:
print('File called %s found. Update it:' % target_file_name)
file_uploaded = update_file(service, target_file_id[0], target_file_name, target_file_descr,
'text/x-script.phyton', target_file_name)
print(str(file_uploaded))
if __name__ == '__main__':
main()
In order to try the example, is necessary to create a Google Drive API from https://console.developers.google.com/apis/dashboard,
then save the file credentials.js and pass its path to get_authenticated(). The file token.json will be created after the first
authentication and API authorization.

The problem is that the metadata 'id' can not be changed when updating a file, so it should not be in the body. Just delete it from the dict:
# File's new metadata.
del file['id'] # 'id' has to be deleted
file['name'] = new_name
file['description'] = new_description
file['mimeType'] = new_mime_type
file['trashed'] = True
I tried your code with this modification and it works

I also struggled a little bit with the function and found if you don't have to update the metadata then just remove them in the update function like :updated_file = service.files().update(fileId=file_id, media_body=media_body).execute()
At Least that worked for me

The problem is The resource body includes fields which are not directly writable. So try removing all of the metadata properties and then add them back one by one. The one I would be suspicious about is trashed. Even though the API docs say this is writable, it shouldn't be. Trashing a file has side effects beyond setting a boolean. Updating a file and setting it to trashed at the same time is somewhat unusual. Are you sure that's what you intend?

Google Drive Python API: Uploading Large Files

I am writing a function to upload a file to Google Drive using the Python API client. It works for files up to 1 MB but does not work for a 10-MB file. When I try to upload a 10-MB file, I get an HTTP 400 error. Any help would be appreciated. Thanks.
Here is the output when I print the error:
An error occurred: <HttpError 400 when requesting https://www.googleapis.com/upload/drive/v3/files?alt=json&uploadType=resumable returned "Bad Request">
Here is the output when I print error.resp:
{'server': 'UploadServer',
'status': '400',
'x-guploader-uploadid': '...',
'content-type': 'application/json; charset=UTF-8',
'date': 'Mon, 26 Feb 2018 17:00:12 GMT',
'vary': 'Origin, X-Origin',
'alt-svc': 'hq=":443"; ma=2592000; quic=51303431; quic=51303339; quic=51303338; quic=51303337; quic=51303335,quic=":443"; ma=2592000; v="41,39,38,37,35"',
'content-length': '171'}
I'm unable to interpret this error. I have tried looking at the Google API Error Guide, but their explanation doesn't make sense to me, as all the parameters are the same as those in the requests with smaller files, which work.
Here is my code:
def insert_file_only(service, name, description, filename='', parent_id='root', mime_type=GoogleMimeTypes.PDF):
""" Insert new file.
Using documentation from Google Python API as a guide:
https://developers.google.com/api-client-library/python/guide/media_upload
Args:
service: Drive API service instance.
name: Name of the file to create, including the extension.
description: Description of the file to insert.
filename: Filename of the file to insert.
parent_id: Parent folder's ID.
mime_type: MIME type of the file to insert.
Returns:
Inserted file metadata if successful, None otherwise.
"""
# Set the file meta data
file_metadata = set_file_metadata(name, description, mime_type, parent_id)
# Create media with correct chunk size
if os.stat(filename).st_size <= 256*1024:
media = MediaFileUpload(filename, mimetype=mime_type, resumable=True)
else:
media = MediaFileUpload(filename, mimetype=mime_type, chunksize=256*1024, resumable=True)
file = None
status = None
start_from_beginning = True
num_temp_errors = 0
while file is None:
try:
if start_from_beginning:
# Start from beginning
logger.debug('Starting file upload')
file = service.files().create(body=file_metadata, media_body=media).execute()
else:
# Upload next chunk
logger.debug('Uploading next chunk')
status, file = service.files().create(
body=file_metadata, media_body=media).next_chunk()
if status:
logger.info('Uploaded {}%'.format(int(100*status.progress())))
except errors.HttpError as error:
logger.error('An error occurred: %s' % error)
logger.error(error.resp)
if error.resp.status in [404]:
# Start the upload all over again
start_from_beginning = True
elif error.resp.status in [500, 502, 503, 504]:
# Increment counter on number of temporary errors
num_temp_errors += 1
if num_temp_errors >= NUM_TEMP_ERROR_LIMIT:
return None
# Call next chunk again
else:
return None
permissions = assign_permissions(file, service)
return file
UPDATE
I tried using a simpler pattern, taking the advice from #StefanE. However, I still get an HTML 400 error for files over 1 MB. New code looks like this:
request = service.files().create(body=file_metadata, media_body=media)
response = None
while response is None:
status, response = request.next_chunk()
if status:
logger.info('Uploaded {}%'.format(int(100*status.progress()))
UPDATE 2
I found that the issue is conversion of the file into a Google Document, not uploading it. I'm trying to upload an HTML file and convert it into a Google Doc. This works for files less than ~2 MB. When I only upload the HTML file but not try to convert it, I don't get the abovementioned error. Looks like this corresponds with the limit on this page. I don't know if this limit can be increased.

I see some issues with your code.
First you have a while loop to continue as long file is None and the first thing you do is to set the value of file. i.e it will only loop once.
Secondly you got variable start_from_beginning but that is never set to False anywhere in the code, the else part of the statement will never be executed.
Looking at the Googles documentation their sample code looks a lot more straight forward:
media = MediaFileUpload('pig.png', mimetype='image/png', resumable=True)
request = farm.animals().insert(media_body=media, body={'name': 'Pig'})
response = None
while response is None:
status, response = request.next_chunk()
if status:
print "Uploaded %d%%." % int(status.progress() * 100)
print "Upload Complete!"
Here you loop on while response is None which will be None until finished with the upload.

Upload multiple files using simple-salesforce python

I started learning SalesForce and developing apps using django.
I need assistance with uploading a file to salesforce, For that I read simple-salesforce and this that help to upload file using rest and SOAP api.
My question is how do I upload one or more files using simple-salesforce?

Here is the code block I use for uploading files.
def load_attachments(sf, new_attachments):
'''
Method to attach the Template from the Parent Case to each of the children.
#param: new_attachments the dictionary of child cases to the file name of the template
'''
url = "https://" + sf.get_forced_url() + ".my.salesforce.com/services/data/v29.0/sobjects/Attachment/"
bearer = "Bearer " + sf.get_session_id()
header = {'Content-Type': 'application/json', 'Authorization': bearer}
for each in new_attachments:
body = ""
long_name = str(new_attachments[each]).split(sep="\\")
short_name = long_name[len(long_name) - 1]
with open(new_attachments[each], "rb") as upload:
body = base64.b64encode(upload.read())
data = json.dumps({
'ParentId': each,
'Name': short_name,
'body': body
})
response = requests.post(url, headers=header, data=data)
print(response.text)
Basically, to send the file, you need to use the requests module and submit the file via a post transaction. The post transaction requires the URL to which the request is sent, the header information, and the data.
Here, sf is the instance of returned by the simple-salesforce initialization. Since my instance uses custom domains, I had to create my own function in simple-salesforce to handle that; I call it get_forced_url(). Note: The URL is may be different for you depending on which version you are using [the v29.0 portion may change].
Then I set up my bearer and header.
The next thing is a loop that submits a new attachment for each attachment in a map from Parent ID to the File I wish to upload. This is important to note, attachments must have a Parent Object so you need to know the ParentId. For each attachment, I blank out the body, create a long and short name for the attachment. Then the important part. On attachments, the actual data of the file is stored as a base-64 binary array. So the file must be opened as binary, hence the "rb" and then encoded to base-64.
Once the file has been parsed to base-64 binary, I build my json string where ParentId is the object ID of the parent object, the Name is the short name, and the body is the base-64 encoded string of data.
Then the file is submitted to the URL with the headers and data. Then I print the response so I could watch it happening.

To upload files, you only need simple-salesforce
Complete example, including creating Account, Contact and Case. Then attaching the file to Case.
#Create Account, Contact and Case
AccountID = sf.Account.create({'Name':'Test12','Phone':'987654321'})["id"]
ContactID = sf.Contact.create({'LastName':'Smith2','Email':'example3#example.com'})["id"]
CaseID = sf.Case.create({'AccountId':AccountID,'ContactId':ContactID,'Description':'Test4321','Subject':'Test4321'})
#Convert image to Base64
import json, base64
with open('test1.png', mode='rb') as file:
img = file.read()
image = base64.encodebytes(img).decode('utf-8')
#The simple example
sf.Attachment.create({'ParentId': CaseID["id"],'Name':'TestFile1','body': image,'ContentType':'image/png'})
And how to change the 'one-file' example to multiple files
sf.bulk.Attachment.insert([
{'ParentId': CaseID["id"],'Name':'TestFile2','body': image,'ContentType':'image/png'},
{'ParentId': CaseID["id"],'Name':'TestFile3','body': image,'ContentType':'image/png'},
{'ParentId': CaseID["id"],'Name':'TestFile4','body': image,'ContentType':'image/png'},
{'ParentId': CaseID["id"],'Name':'TestFile5','body': image,'ContentType':'image/png'},
{'ParentId': CaseID["id"],'Name':'TestFile6','body': image,'ContentType':'image/png'},
{'ParentId': CaseID["id"],'Name':'TestFile7','body': image,'ContentType':'image/png'},
{'ParentId': CaseID["id"],'Name':'TestFile8','body': image,'ContentType':'image/png'},
{'ParentId': CaseID["id"],'Name':'TestFile9','body': image,'ContentType':'image/png'}],batch_size=1000,use_serial=True)
(you know how to fix the rest)

Deleting dataset attachment in Socrata with API

I'm working on a python script that will:
1) pull GIS metadata from an enterprise database
2) parse the metadata from XML to plain text
3) attach the text files to the corresponding published datasets in Socrata (which are published monthly)
4) The script will also be run monthly, so that any schema changes in the enterprise dataset are reflected in the attached plain text metadata files on Socrata.
I've been able to successfully attach the text metadata files to published Socrata datasets using some code found here. The problem is, each time the script is run, an additional attachment is added. I would like to either delete the existing attachment and add a new one, or overwrite the existing attachment with the contents of the new one.
I've done a fair amount of research on this, and can't seem to find any documentation for managing attachments using the Socrata API. Any suggestions?

I ended up figuring this one out. Had to alter a few lines to empty out the attachments in the attach_file function in the Socrata Python library from:
def attach_file(self, filename):
metadata = self.metadata()
if not metadata.has_key('attachments'):
metadata['attachments'] = []
to:
def attach_file(self, filename):
metadata = self.metadata()
metadata['attachments'] = [] #empty out metadata, regardless of existing metadata

Ended up using the same API and was able to replace attachments using the following code:
def attach_file(self, filename, clear_metadata):
metadata = self.metadata()
if not metadata.has_key('attachments'):
metadata['attachments'] = []
# if the user wants to clear all existing attachments on dataset
if clear_metadata:
metadata['attachments'] = []
response = self.multipart_post('/assets', filename)
if not response.has_key('id'):
print "Error uploading file to assets service: no ID returned: %s" % response
return
attachment = {'blobId': response['id'],
'name': response['nameForOutput'],
'filename': response['nameForOutput']}
metadata['attachments'].append(attachment)
self._request("/views/%s.json" % self.id, 'PUT', {'metadata':metadata})
def multipart_post(self, url, filename, field='file'):
print("Running multipart_post")
authBase64 = base64.encodestring('%s:%s' % (self.username, self.password)).replace('\n', '')
datagen, headers = multipart_encode({field: open(filename, "rb")})
headers['X-App-Token'] = self.app_token
headers['Authorization'] = "Basic %s" % authBase64
print("url=" + url)
request = Request("%s%s" % (self.url, url), datagen, headers)
print(str(Request))
response = urlopen(request).read()
return json.loads(response)

JIRA Python add_attachment() 405 Method Not Allowed

I am trying to upload a file to JIRA via its REST API using the python lib found here: jira python documentation
It seems pretty straight forward I wrote a method that allows me to pass an issue and then it attaches a filename. and one that lets me retrieve an issue from JIRA.
from jira.client import JIRA
class JIRAReport (object):
def attach(self,issue):
print 'Attaching... '
attachment = self.jira.add_attachment(issue, attachment=self.reportpath, filename='Report.xlsx')
print 'Success!'
def getissue(self):
if not self.issue == None:
return self.jira.issue(self.issue)
return None
then in my main script I am getting the issue and attaching the file to an issue I retrieved from JIRA
report = JiraReport()
report.issue = 'ProjectKey-1'
report.reportpath = '../report_upload/tmp/' + filename
issue = report.getissue()
if not issue == None:
report.attach(issue)
else:
print "No Issue with Key Found"
I am able to get the issue/create issues if needed but when using the self.jira.add_attachment() method I am getting 405 Method Not Allowed.
The file exists and is able to be opened.
Here is the add_attachment() method from the source code:
def add_attachment(self, issue, attachment, filename=None):
"""
Attach an attachment to an issue and returns a Resource for it.
The client will *not* attempt to open or validate the attachment; it expects a file-like object to be ready
for its use. The user is still responsible for tidying up (e.g., closing the file, killing the socket, etc.)
:param issue: the issue to attach the attachment to
:param attachment: file-like object to attach to the issue, also works if it is a string with the filename.
:param filename: optional name for the attached file. If omitted, the file object's ``name`` attribute
is used. If you aquired the file-like object by any other method than ``open()``, make sure
that a name is specified in one way or the other.
:rtype: an Attachment Resource
"""
if isinstance(attachment, string_types):
attachment = open(attachment, "rb")
# TODO: Support attaching multiple files at once?
url = self._get_url('issue/' + str(issue) + '/attachments')
fname = filename
if not fname:
fname = os.path.basename(attachment.name)
content_type = mimetypes.guess_type(fname)[0]
if not content_type:
content_type = 'application/octet-stream'
files = {
'file': (fname, attachment, content_type)
}
r = self._session.post(url, files=files, headers=self._options['headers'])
raise_on_error(r)
attachment = Attachment(self._options, self._session, json.loads(r.text)[0])
return attachment

It is mentioned in documentation that as a argument they expect file-like object.
Try to do something like :
file_obj = open('test.txt','rb')
jira.add_attachment(issue,file_obj,'test.txt')
file_obj.close()

Check that the URL that you are specifying for JIRA (if using the on-demand service) is https://instance.atlassian.net.
I just hit this as well, and it sends a POST request to http://instance.atlassian.net and gets redirected to https://instance.atlassian.net, but the client sends a GET request to the redirected address (see: https://softwareengineering.stackexchange.com/questions/99894/why-doesnt-http-have-post-redirect for more information)