I'm trying to upload files > 5mb to google drive with python over the rest api, so i'm using the chunked upload method.
When the file is small enough to fit in one request everything is fine, but when i have a larger file it fails at the last request with
header i sended: "{'Content-Length': '416768', 'Content-Type': 'application/tar', 'Content-Range': 'bytes 262144-416768/416768'}"
content:"Failed to parse Content-Range header."
And yes this file is only 407kb big to have faster test execution (20mb file also fails).
This is my code made from this SO Question for the sessionUri, file_name=backup.tar.
The imported google_api is my script witch returns folder_id and access_token.
It finishes successfully and return's the sessioUri.
file_size = str(os.path.getsize(file_name))
import google_api, base64
from datetime import datetime
access_token = google_api.access_token()
folder_id = google_api.folder_id()
headers = {
'Authorization': 'Bearer ' + access_token,
'Content-Type': 'application/json; charset=UTF-8',
'X-Upload-Content-Type': 'application/tar',
'X-Upload-Content-Length': file_size,
'X-Upload-Content-Range': 'bytes 0-{}/{}'.format(file_size, file_size)
}
querystring = {"uploadType": "resumable"}
THUMBNAIL_IMAGE = str(open("drive_image.base64","r").read())
THUMBNAIL_MIME_TYPE = "image/png"
f = open("drive_logo.png","rb")
urlsafe_thumbnail = base64.urlsafe_b64encode(f.read()).decode('utf8')
drive_content_hints = '"contentHints": {"thumbnail": { "image": "'+urlsafe_thumbnail+'", "mimeType": "image/png" } } '
drive_parents_folder = '"parents": ["'+ folder_id +'"]'
drive_description = '"description": "This is a backup file"'
drive_file_name = '"name": "'+file_name+'"'
currentdate = datetime.strftime(datetime.today(),"%Y-%m-%dT%H:%M:%SZ")
drive_creation_time = '"createdTime": "'+currentdate+'"'
drive_modificition_time = '"createdTime": "'+currentdate+'"'
payload = '{'+drive_file_name+','+drive_description+','+drive_content_hints+','+drive_creation_time+','+drive_modificition_time+','+drive_parents_folder+'}'
response = requests.post(
'https://www.googleapis.com/upload/drive/v3/files',
headers=headers,
data=payload,
params=querystring
)
if response.status_code == 200:
sessionUri = response.headers['Location']
else:
print("error")
print(response.content)
return
The problem is with the rest of the code.
file_size = os.path.getsize(file_name)
in_file = open(file_name, "rb") # opening for [r]eading as [b]inary
chunk = in_file.read()
print(file_size)
BASE_CHUNK_SIZE = 256 * 1024 # 262144
CHUNK_SIZE = 1 * BASE_CHUNK_SIZE
TOTAL_BYTES = file_size
first_byte = 0
last_byte = CHUNK_SIZE - 1
import math
times = int(math.ceil(file_size/CHUNK_SIZE))
print(times)
for _ in range(times):
if last_byte > TOTAL_BYTES:
last_byte = TOTAL_BYTES -1
print("hi")
data2 = chunk[first_byte:last_byte+1]
bytes = "bytes 0-" + str(CHUNK_SIZE-1) + "/" + str(file_size)
headers = {
'Content-Length': str(TOTAL_BYTES),
'Content-Type': "application/tar",
'Content-Range': "bytes " + str(first_byte) +"-"+str(last_byte)+"/"+str(TOTAL_BYTES)
}
print(headers)
response = requests.request(
"PUT", sessionUri, data=data2, headers=headers)
print(response.content)
byte_range = response.headers["Range"]
first_byte = byte_range.split("=",1)[1].split("-",1)[0]
last_byte = byte_range.split("=",1)[1].split("-",1)[1]
print(last_byte)
first_byte = int(last_byte)+1
last_byte = int(first_byte)+CHUNK_SIZE
print(str(first_byte)+"-"+str(last_byte))
I appreciate your help.
SOLVED IT! The if for the last request must contain -1 because it gets added later. Corrected my question! Need to add END for RANGE header after last request.
Related
I am trying to upload reel with Graph Api using Python.
I'm getting an error every time I try to upload video.
Error :
{"debug_info":{"retriable":false,"type":"NotAuthorizedError","message":"User not authorized to perform this request"}}
Note: I have given every possible permission to my app and page.
code:
import requests
import os
import json
Title ="Title of the video"
title = Title
description = title
source = f"F:\proj_ytTofb\downloads\{Title}.mp4"
files = {'source': open(source, 'rb')}
file_size = os.path.getsize(source)
print("File Size is :", file_size, "bytes")
def Initialize():
url = f"https://graph.facebook.com/v13.0/{page_id}/video_reels?upload_phase=start"
payload = {
'access_token': token,
}
r = requests.post(url, data = payload)
return r.json()["video_id"]
video_id=Initialize()
print(video_id)
def Upload():
url = f"https://rupload.facebook.com/video-upload/v15.0/{video_id}"
payload ={
'access_token': token,
'offset': 0,
'file_size': file_size,
}
r = requests.post(url, files=files, data=payload)
return r.text
print(Upload())
output: {"debug_info":{"retriable":false,"type":"NotAuthorizedError","message":"User not authorized to perform this request"}}
Your Upload code seem bit wrong if you refer on documentation
def Upload(vidid, size, filedata):
url = f"https://rupload.facebook.com/video-upload/v13.0/{vidid}"
payloadUp = {
'Authorization': 'OAuth ' + page_access_token,
'offset': "0",
'file_size': str(size),
}
print(payloadUp)
r = requests.post(url, data=filedata, headers=payloadUp)
return r.text
with parameter like this
files = {'source': open(mp4_path, 'rb')}
file_size = os.path.getsize(mp4_path)
and then you called it like this
Upload(video_id, file_size, files)
Note: I successfully upload it on fb reels and published it, but I dont what happen the video failed to convert without error notice.
I have been trying to export offer file using Python for Bol Retailer API
According to the official docs on request an offer file export
I have include all the headers and formats but it throws a 400 Bad Request
400
Bad Request
b'{\n "type" : "https://api.bol.com/problems",\n "title" : "Bad Request",\n "status" : 400,\n "detail" : "The supplied content-type media type is not supported.",\n "host" : "Instance-111",\n "instance" : "https://api.bol.com/retailer/offers/export"\n}'
Here is a minimal example from my code
import base64
import requests
import json
import time
class BolService:
def __init__(self, _id, secret):
self.host = "https://api.bol.com"
self.__header = {
"Accept": "application/vnd.retailer.v7+json",
"Content-Type": "N/A",
"Authorization": "Bearer " + self.__get_token(_id, secret)
}
def __get_token(self, _id, secret) -> str:
creds = (_id + ":" + secret).encode('ascii') #creds : ascii bytes
creds_b64_b = base64.b64encode(creds) #creds : base64 bytes
creds_b64 = creds_b64_b.decode('ascii') #creds : base64 string
header = {
"Authorization":"Basic " + creds_b64
}
link = "https://login.bol.com/token?grant_type=client_credentials"
response = requests.post(link, headers=header)
response_data = json.loads(response.content.decode())
return response_data['access_token']
def get_offer_file(self):
path = f"/retailer/offers/export"
new_header = self.__header.copy()
new_header["format"] = "CSV"
response = requests.post(self.host + path, headers=new_header)
return response
Note: I have also tried changing the "Content-Type" in self.__header to "application/vnd.retailer.v7+json", I have also changed the same to add csv using "application/vnd.retailer.v7+json+csv" or "application/vnd.retailer.v7+csv". I have also tried adding self.__header['Content-Type'] = 'text/csv' but nothing seems to work it keeps on throwing the same Bad Request. I have also tried using the v6 of the API instead of v7 but same issue.
I know this is something that should be dealt with the customer service of Bol but they their service is too pathetic to even give a simple reply. Also as of August 2022 their site which details API issues is down. Maybe if someone with experience can help here.
I don't think I am missing anything here. Please let me know.
So I was able to sucessfully make the POST request.
1st what I did was change the "Content-Type" in self.__header to "application/vnd.retailer.v7+json"
so the header now looks like this
self.__header = {
"Accept": "application/vnd.retailer.v7+json",
"Content-Type": "application/vnd.retailer.v7+json",
"Authorization": "Bearer " + self.__get_token(_id, secret)
}
Since we require the content type in JSON format so we have to include a JSON body by dumping our dictionary content using json.dumps
So the get_offer_file method now looks like with {"format":"CSV"} as the body
def get_offer_file(self):
path = f"/retailer/offers/export"
response = requests.post(self.host + path, headers=self.__header, data=json.dumps({"format":"CSV"}))
return response
Here is the full code:
import base64
import requests
import json
class BolService:
def __init__(self, _id, secret):
self.host = "https://api.bol.com"
self.__header = {
"Accept": "application/vnd.retailer.v7+json",
"Content-Type": "application/vnd.retailer.v7+json",
"Authorization": "Bearer " + self.__get_token(_id, secret)
}
def __get_token(self, _id, secret) -> str:
creds = (_id + ":" + secret).encode('ascii') #creds : ascii bytes
creds_b64_b = base64.b64encode(creds) #creds : base64 bytes
creds_b64 = creds_b64_b.decode('ascii') #creds : base64 string
header = {
"Authorization":"Basic " + creds_b64
}
link = "https://login.bol.com/token?grant_type=client_credentials"
response = requests.post(link, headers=header)
response_data = json.loads(response.content.decode())
return response_data['access_token']
def get_offer_file(self):
path = f"/retailer/offers/export"
response = requests.post(self.host + path, headers=self.__header, data=json.dumps({"format":"CSV"}))
return response
I've been having some trouble sending files via python's rest module. I can send emails without attachments just fine but as soon as I try and add a files parameter, the call fails and I get a 415 error.
I've looked through the site and found out it was maybe because I wasn't sending the content type of the files when building that array of data so altered it to query the content type with mimetypes; still 415.
This thread: python requests file upload made a couple of more edits but still 415.
The error message says:
"A supported MIME type could not be found that matches the content type of the response. None of the supported type(s)"
Then lists a bunch of json types e.g: "'application/json;odata.metadata=minimal;odata.streaming=true;IEEE754Compatible=false"
then says:
"matches the content type 'multipart/form-data; boundary=0e5485079df745cf0d07777a88aeb8fd'"
Which of course makes me think I'm still not handling the content type correctly somewhere.
Can anyone see where I'm going wrong in my code?
Thanks!
Here's the function:
def send_email(access_token):
import requests
import json
import pandas as pd
import mimetypes
url = "https://outlook.office.com/api/v2.0/me/sendmail"
headers = {
'Authorization': 'Bearer '+access_token,
}
data = {}
data['Message'] = {
'Subject': "Test",
'Body': {
'ContentType': 'Text',
'Content': 'This is a test'
},
'ToRecipients': [
{
'EmailAddress':{
'Address': 'MY TEST EMAIL ADDRESS'
}
}
]
}
data['SaveToSentItems'] = "true"
json_data = json.dumps(data)
#need to convert the above json_data to dict, otherwise it won't work
json_data = json.loads(json_data)
###ATTACHMENT WORK
file_list = ['test_files/test.xlsx', 'test_files/test.docx']
files = {}
pos = 1
for file in file_list:
x = file.split('/') #seperate file name from file path
files['file'+str(pos)] = ( #give the file a unique name
x[1], #actual filename
open(file,'rb'), #open the file
mimetypes.MimeTypes().guess_type(file)[0] #add in the contents type
)
pos += 1 #increase the naming iteration
#print(files)
r = requests.post(url, headers=headers, json=json_data, files=files)
print("")
print(r)
print("")
print(r.text)
I've figured it out! Took a look at the outlook API documentation and realised I should be adding attachments as encoded lists within the message Json, not within the request.post function. Here's my working example:
import requests
import json
import pandas as pd
import mimetypes
import base64
url = "https://outlook.office.com/api/v2.0/me/sendmail"
headers = {
'Authorization': 'Bearer '+access_token,
}
Attachments = []
file_list = ['test_files/image.png', 'test_files/test.xlsx']
for file in file_list:
x = file.split('/') #file the file path so we can get it's na,e
filename = x[1] #get the filename
content = open(file,'rb') #load the content
#encode the file into bytes then turn those bytes into a string
encoded_string = ''
with open(file, "rb") as image_file:
encoded_string = base64.b64encode(image_file.read())
encoded_string = encoded_string.decode("utf-8")
#append the file to the attachments list
Attachments.append({
"#odata.type": "#Microsoft.OutlookServices.FileAttachment",
"Name": filename,
"ContentBytes": encoded_string
})
data = {}
data['Message'] = {
'Subject': "Test",
'Body': {
'ContentType': 'Text',
'Content': 'This is a test'
},
'ToRecipients': [
{
'EmailAddress':{
'Address': 'EMAIL_ADDRESS'
}
}
],
"Attachments": Attachments
}
data['SaveToSentItems'] = "true"
json_data = json.dumps(data)
json_data = json.loads(json_data)
r = requests.post(url, headers=headers, json=json_data)
print(r)
I'm connecting to API which has 500 rows limit per call.
This is my code for a single API call (Works great):
def getdata(data):
auth_token = access_token
hed = {'Authorization': 'Bearer ' + auth_token, 'Accept': 'application/json'}
urlApi = 'https://..../orders?Offset=0&Limit=499'
datar = requests.get(urlApi, data=data, headers=hed, verify=True)
return datar
Now I want to scale it up so it will get me all the records.
This is what I tried to do:
In order to make sure that I have all the rows, I must iterate until there is no more data:
get 1st page
get 2nd page
merge
get 3rd page
merge
etc...
each page is an API call.
This is what I'm trying to do:
def getData(data):
auth_token = access_token
value_offset = 0
hed = {'Authorization': 'Bearer ' + auth_token, 'Accept': 'application/json'}
datarALL = None
while True:
urlApi = 'https://..../orders?Offset=' + value_offset + '&Limit=499'
responsedata = requests.get(urlApi, data=data, headers=hed, verify=True)
if responsedata.ok:
value_offset = value_offset + 499
#to do: merge the result of the get request
datarALL= datarALL+ responsedata (?)
# to do: check if response is empty then break out.
return datarALL
I couldn't find information about how I merge the results of the API calls nor how do I check if I can break the loop.
Edit:
To clear what I'm after.
I can see the results of the API call using:
logger.debug('response is : {0}'.format(datar.json()))
What I want to be able to do:
logger.debug('response is : {0}'.format(datarALL.json()))
and it will show all results from all calls. This requires generate API calls until there is no more data to get.
This is the return sample of API call:
"offset": 0,
"limit": 0,
"total": 0,
"results": [
{
"field1": 0,
"field2": "string",
"field3": "string",
"field4": "string"
}
]
}
In this case, you are almost correct with the idea.
is_valid = True
while is_valid:
is_valid = False
...
...
responsedata = requests.get(urlApi, data=data, headers=hed, verify=True)
if responsedata.status_code == 200: #Use status code to check request status, 200 for successful call
responsedata = responsedata.text
value_offset = value_offset + 499
#to do: merge the result of the get request
jsondata = json.loads(responsedata)
if "results" in jsondata:
if jsondata["results"]:
is_valid = True
if is_valid:
#concat array by + operand
datarALL = datarALL + jsondata["results"]
As I don't know if "results" still exists when the data ran out, so I checked both level.
I have a python script where I am sending a POST request for data to a server. I am expecting a particular response which indicates there is data in the response. If I do not receive this response, how can I restart my script/go to the beginning of it. The script is wrapped in a function which allows it to run every minute.
I would like to return to the beginning of my function if my response isn't as expected.
Script:
import sched, time, requests, jsonpickle, arcpy, requests, json, datetime
s = sched.scheduler(time.time, time.sleep)
def do_something(sc):
data2 = jsonpickle.decode((f2.read()))
Start = datetime.datetime.now()
# Start = datetime.datetime.strftime(data2['QueryRequest']['LastUpdatedDate'])
DD = datetime.timedelta(minutes=5)
earlier = Start - DD
earlier_str = earlier.strftime('X%m/%d/%Y %H:%M:%S').replace('X0','X').replace('X','')
data2["QueryRequest"]['LastUpdatedDate'] = str(earlier_str)
data2 = jsonpickle.encode(data2)
BulkyItemInfo = " "
spatial_ref = arcpy.SpatialReference(4326)
lastpage = 'false'
startrow = 0
newquery = 'new'
pagesize = 100
url2 = "URL"
headers2 = {'Content-type': 'text/plain', 'Accept': '/'}
while lastpage == 'false':
r2 = requests.post(url2, data=data2, headers=headers2)
print r2.text
decoded2 = json.loads(r2.text)
f2 =open('C:\Users\GeoffreyWest\Desktop\Request.json')
data2 = jsonpickle.decode((f2.read()))
if decoded2['Response']['LastPage'] == 'false':
data2['QueryRequest']['PageSize'] = pagesize
startrow = startrow + data2['QueryRequest']['PageSize']
data2['QueryRequest']['StartRowNum'] = startrow
data2['QueryRequest']['NewQuery'] = 'false'
data2 = jsonpickle.encode(data2)
print startrow
else:
lastpage = 'true'
print json.dumps(decoded2, sort_keys=True, indent=4)
items = []
for sr in decoded2['Response']['ListOfServiceRequest']['ServiceRequest']:#Where response is successful or fails
Output for successful response:
{
"status": {
"code": 311,
"message": "Service Request Successfully Queried.",
"cause": ""
},
"Response": {
"LastPage": "false",
"NumOutputObjects": "100",
"ListOfServiceRequest": {
"ServiceRequest": [
{
Output for unsuccessful response:
{"status":{"code":311,"message":"Service Request Successfully Queried.","cause":""},"Response":{"LastPage":"true","NumOutputObjects":"0","ListOfServiceRequest":{}}}