I have code that uploads my archive to Google Drive using my access token and requests, but if the file is larger than 512MB, it will fail with exit code MemoryError, so I'm searching for a way to fix this error and upload a file larger than 512MB. I already tried to find a solution but I didn't find anything where I could use an access token.
import os
import json
import requests
import ntpath
import oauth2
import httplib2
import oauth2client
from contextlib import closing
from googleapiclient.discovery import build
from oauth2client.client import GoogleCredentials
_CLIENT_ID = 'YOUR_CLIENT_ID'
_CLIENT_SECRET = 'YOUR_CLIENT_SECRET'
_REFRESH_TOKEN = 'YOUR_REFRESH_TOKEN'
_PARENT_FOLDER_ID = 'YOUR_PARENT_FOLDER_ID'
_ARCHIVE_FILE = os.environ['USERPROFILE'] +'\\Desktop\\WobbyChip.zip'
# ====================================================================================
def GetAccessToken(client_id, client_secret, refresh_token):
cred = oauth2client.client.GoogleCredentials(None,client_id,client_secret,refresh_token,None,'https://accounts.google.com/o/oauth2/token',None)
http = cred.authorize(httplib2.Http())
cred.refresh(http)
obj = json.loads(cred.to_json())
_ACCESS_TOKEN = obj['access_token']
return _ACCESS_TOKEN
def UploadFile(local_file, parent_folder_id, access_token,):
headers = {'Authorization': 'Bearer ' +access_token}
para = {
'name': (ntpath.basename(local_file)),
'parents': [parent_folder_id]}
files = {
'data': ('metadata', json.dumps(para), 'application/json; charset=UTF-8'),
'file': open(local_file, 'rb')}
requests.post(
'https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart',
headers=headers,
files=files)
# ====================================================================================
if __name__ == '__main__':
UploadFile(_ARCHIVE_FILE, _PARENT_FOLDER_ID, GetAccessToken(_CLIENT_ID, _CLIENT_SECRET, _REFRESH_TOKEN))
Related
I have a model endpoint running in Azure Kubernetes Service and I am not using Django or Flask. I am sending local png files to score as follows:
import base64
import json
import cv2
import requests
img_path = 'C:/path/to/exampleImage.png'
link = aks_service.scoring_uri
api_key = aks_service.get_keys()[0]
def send2score(img_path, score_url, api_key):
headers = {'Content-Type': 'application/json',
'Authorization': ('Bearer ' + api_key)
}
img = cv2.imread(img_path)
string = base64.b64encode(cv2.imencode('.png', img)[1]).decode()
dict = {
'img': string
}
jsonimg2 = json.dumps(dict, ensure_ascii=False, indent=4)
resp = requests.post(url=link, data=jsonimg2, headers=headers)
print(resp.text)
send2score(img_path=img_path, score_url=link, api_key=api_key)
My question is: how can I get the file name (exampleImage.png) in the score script in Azure Kubernetes after I do the request.post? Please no Django or Flask specific methods
Bonus question: Feel free to suggest improvements in the way I am uploading the data (send2score function), this function is working, I get the score back, I just can't get the file name in the score script. Thank you!
According to your code, you send your image as base64 string. It cannot contain the file name. I think you need to define a parameter to store the file name in request body. Besides, you also can post the file as a Multipart-Encoded File with the requests module.
For example
send file
import requests
import magic
import os
url = ''
path = "D:/sample/image/faces.jpg"
mime = magic.Magic(mime=True)
headers = {
'Authorization': ('Bearer ' + 'cdff')
}
files = {'file': (os.path.basename(path), open(path, 'rb'), mime.from_file(path), {'Expires': '0'})}
res = requests.post(url, files=files, headers=headers)
print(res.content.decode('utf-8'))
My backend
from http.server import BaseHTTPRequestHandler, HTTPServer
import cgi
hostName =
hostPort =
class MyServer(BaseHTTPRequestHandler):
def do_POST(self):
try:
// use cgi to read file
form = cgi.FieldStorage(fp=self.rfile, headers=self.headers, environ={
'REQUEST_METHOD': 'POST', 'CONTENT_TYPE': self.headers['Content-Type'], })
file = form.list[0]
data =file.file.read()
#process data
.............
self.send_response(200)
self.send_header("Content-type", "text/html")
self.end_headers()
self.wfile.write(bytes(
f"<html><head></head><body><h2>fileName : {file.filename}</h2>/html>", "utf-8"))
except Exception as e:
httperror = e.httperror if hasattr(e, "httperror") else 500
self.send_error(httperror, str(e)) # Send an error response
myServer = HTTPServer((hostName, hostPort), MyServer)
myServer.serve_forever()
I was overcomplicating things, I realized that I am sending the encoded image as a json in a dictionary. I can include other information, like the file name, in the dictionary:
import base64
import json
import cv2
import requests
img_path = 'C:/path/to/exampleImage.png'
link = aks_service.scoring_uri
api_key = aks_service.get_keys()[0]
def send2score(img_path, score_url, api_key):
headers = {'Content-Type': 'application/json',
'Authorization': ('Bearer ' + api_key)
}
img = cv2.imread(img_path)
string = base64.b64encode(cv2.imencode('.png', img)[1]).decode()
dict = {
'imgname': os.path.basename(img_path),
'img': string
}
jsonimg2 = json.dumps(dict, ensure_ascii=False, indent=4)
resp = requests.post(url=link, data=jsonimg2, headers=headers)
print(resp.text)
send2score(img_path=img_path, score_url=link, api_key=api_key)
And I can get the image and file name in the score script:
# Lots of code before
response = json.loads(path)
string = response['img']
jpg_original = base64.b64decode(string) # decode
jpg_as_np = np.frombuffer(jpg_original, dtype=np.uint8)
img0 = cv2.imdecode(jpg_as_np, flags=1) # image
img0name = response['imgname'] # file name
# Lots of code after
I am trying to upload my excel spreadsheet to a document library on my SharePoint Online site. The Sharepoint URL and the folder location on the SharePoint site are listed in the excel Spreadsheet.
Here is the code that I have right now:
import numpy as np
import pandas as pd
import xlwings as xw
from xlwings.constants import Direction
import sys
import requests
from requests_ntlm import HttpNtlmAuth
pd.options.mode.chained_assignment = None
def Upload():
wb = xw.Book.caller()
ws = wb.sheets['Sheet1']
#Read filename from excel
fileName = sys.argv[1]
#Enter SharePoint ONline site and target library
SP_URL = ws.range('C7').value
folder_URL = ws.range('C8').value
#Set up the url for requesting file upload
request_URL = SP_URL + '/_api/web/getfolderbyserverrelativeurl(\'' +
folder_URL + '\')/Files/asdd(url=\'' + fileName + '\',overwrite=true)'
#read in the file that we are going to upload
file = open(fileName, 'rb')
headers = {'Content-Type': 'application/json; odata=verbose', 'accept':
'application/json;odata=verbose'}
r = requests.post(SP_URL +
"/_api/contextinfo",auth=HttpNtlmAuth('Domain\\username','password'),
headers=headers)
formDigestValue = r.json()['d']['GetContextWebInformation']
['FormDigestValue']
headers = {'Content-Type': 'application/json; odata=verbose', 'accept':
'application/json;odata=verbose', 'x-requestdigest' : formDigestValue}
uploadResult =
requests.post(request_URL,auth=HttpNtlmAuth('Domain\\username','password'),
headers=headers, data=file.read())
I am receiving the following error:
formDigestValue = r.json()['d']['GetContextWebInformation']['FormDigestValue']
KeyError: 'd'
requests_ntlm package
allows for HTTP NTLM authentication using the requests library
but NTLM is not supported for SharePoint Online.
Instead of requests_ntlm i would suggest to utilize Office365-REST-Python-Client (it supports to specify user credentials and consumes SharePoint REST API) package to upload file into SharePoint Online, for example:
ctx_auth = AuthenticationContext(url=settings['url'])
if ctx_auth.acquire_token_for_user(username=settings['user_credentials']['username'],
password=settings['user_credentials']['password']):
ctx = ClientContext(settings['url'], ctx_auth)
target_list = ctx.web.lists.get_by_title("Documents")
info = FileCreationInformation()
file_name = "Book.xlsx"
path = "{0}/data/{1}".format(os.path.dirname(__file__), file_name)
with open(path, 'rb') as content_file:
info.content = content = content_file.read()
info.url = file_name
info.overwrite = True
upload_file = target_list.root_folder.files.add(info)
ctx.execute_query()
formDigestValue = r.json()['d']['GetContextWebInformation']['FormDigestValue']
KeyError: 'd'
All this means is that the response content doesn't have 'd' as a key. Try looking at the json code
print(r.content) or something, there could be an error message indicating what is wrong with your post request
I have a problem statement where I have to login to a website and then download a zip file. I have written the below code so far to login to website(able to print authentication successful message) and create a session. How can I download the zip file now ?
import requests
import urllib
import urllib.request
import zipfile
import io
import shutil
post_login_url = 'https://www.ims-dm.com/mvc/page/customer-sign-in/cgi/cookie.php'
request_url = 'http://www.ims-dm.com/cgi/securedownload.php?p=WPNFTPD#prodtype=wpn/WPN-FULL-20180306.TXT.zip'
payload = {
'sendusername':'xxxxxxxxxx',
'password':'xxxxxx'
}
with requests.Session() as session:
post = session.post(post_login_url,data=payload)
if post.status_code == 200:
print("Authentication sucessful !!")
url = session.get(request_url)
I'm trying to upload a file to OneDrive of an MS SharePoint using Python API. I've succeeded in authentication, but I can't get any service_info returned.
I've already looked here and here. I'm experiencing the exact same issues describe there. I've modified my code accordingly, but nothing seems to work.
Here's my complete code.
import onedrivesdk
from onedrivesdk.helpers import GetAuthCodeServer
from onedrivesdk.helpers.resource_discovery import ResourceDiscoveryRequest, ServiceInfo
from onedrivesdk.helpers import http_provider_with_proxy
redirect_uri = 'http://localhost:8080'
client_id='xxxxxxx'
client_secret = 'xxxxxxx'
discovery_uri = 'https://api.office.com/discovery/'
auth_server_url='https://login.microsoftonline.com/common/oauth2/authorize'
auth_token_url='https://login.microsoftonline.com/common/oauth2/token'
proxy = {
'http': 'xxx',
'https': 'xxx'
}
import json
import requests
class AnyVersionResourceDiscoveryRequest(ResourceDiscoveryRequest):
def get_all_service_info(self, access_token, sharepoint_base_url):
headers = {'Authorization': 'Bearer ' + access_token}
response = json.loads(requests.get(self._discovery_service_url,
headers=headers).text)
service_info_list = [ServiceInfo(x) for x in response['value']]
# Get all services, not just the ones with service_api_version 'v2.0'
# Filter only on service_resource_id
sharepoint_services = \
[si for si in service_info_list
if si.service_resource_id == sharepoint_base_url]
return sharepoint_services
sharepoint_base_url = 'https://xxx.sharepoint.com/'
# http = onedrivesdk.HttpProvider()
http = http_provider_with_proxy.HttpProviderWithProxy(proxy, verify_ssl=True)
auth = onedrivesdk.AuthProvider(http,
client_id,
auth_server_url=auth_server_url,
auth_token_url=auth_token_url)
auth_url = auth.get_auth_url(redirect_uri)
code = GetAuthCodeServer.get_auth_code(auth_url, redirect_uri)
auth.authenticate(code, redirect_uri, client_secret, resource=discovery_uri)
# If you have access to more than one service, you'll need to decide
# which ServiceInfo to use instead of just using the first one, as below.
# service_info = ResourceDiscoveryRequest().get_service_info(auth.access_token)[0]
service_info = AnyVersionResourceDiscoveryRequest().get_all_service_info(auth.access_token, sharepoint_base_url)[0]
auth.redeem_refresh_token(service_info.service_resource_id)
client = onedrivesdk.OneDriveClient(service_info.service_resource_id + '/_api/v2.0/', auth, http)
I'm stuck at the 3rd line from the bottom. What am I doing wrong here?
Is there anyway I can analyse URL's using Google Cloud Vision. I know how to analyse images that I store locally, but I can't seem to analyse jpg's that exist on the internet:
import argparse
import base64
import httplib2
from googleapiclient.discovery import build
import collections
import time
import datetime
import pyodbc
time_start = datetime.datetime.now()
def main(photo_file):
'''Run a label request on a single image'''
API_DISCOVERY_FILE = 'https://vision.googleapis.com/$discovery/rest?version=v1'
http = httplib2.Http()
service = build('vision', 'v1', http, discoveryServiceUrl=API_DISCOVERY_FILE, developerKey=INSERT API KEY HERE)
with open(photo_file, 'rb') as image:
image_content = base64.b64encode(image.read())
service_request = service.images().annotate(
body={
'requests': [{
'image': {
'content': image_content
},
'features': [{
'type': 'LOGO_DETECTION',
'maxResults': 10,
}]
}]
})
response = service_request.execute()
try:
logo_description = response['responses'][0]['logoAnnotations'][0]['description']
logo_description_score = response['responses'][0]['logoAnnotations'][0]['score']
print logo_description
print logo_description_score
except KeyError:
print "logo nonexistent"
pass
print time_start
if __name__ == '__main__':
main("C:\Users\KVadher\Desktop\image_file1.jpg")
Is there anyway I can analyse a URL and get an answer as to whether there are any logo's in them?
I figured out how to do it. Re-wrote my code and added used urllib to open the image and then I passed it through base64 and the google cloud vision logo recognition api:
import argparse
import base64
import httplib2
from googleapiclient.discovery import build
import collections
import time
import datetime
import pyodbc
import urllib
import urllib2
time_start = datetime.datetime.now()
#API AND DEVELOPER KEY DETAILS
API_DISCOVERY_FILE = 'https://vision.googleapis.com/$discovery/rest?version=v1'
http = httplib2.Http()
service = build('vision', 'v1', http, discoveryServiceUrl=API_DISCOVERY_FILE, developerKey=INSERT DEVELOPER KEY HERE)
url = "http://www.lcbo.com/content/dam/lcbo/products/218040.jpg/jcr:content/renditions/cq5dam.web.1280.1280.jpeg"
opener = urllib.urlopen(url)
#with open(photo_file) as image:
image_content = base64.b64encode(opener.read())
service_request = service.images().annotate(
body={
'requests': [{
'image': {
'content': image_content
},
'features': [{
'type': 'LOGO_DETECTION',
'maxResults': 10,
}]
}]
})
response = service_request.execute()
try:
logo_description = response['responses'][0]['logoAnnotations'][0]['description']
logo_description_score = response['responses'][0]['logoAnnotations'][0]['score']
print logo_description
print logo_description_score
except KeyError:
print "logo nonexistent"
pass
print time_start
The Google Cloud Vision API allows you to either specify the image content in base64 or a link to a file on Google Cloud storage. See:
https://cloud.google.com/vision/docs/requests-and-responses#json_request_format
This means that you will have to download each image url in your code (using Python's urllib2 library maybe) and encode it in base64, then add it to service_request.