I am using the following code in my windows which works but as soon as I deploy it as cloud function on GCP I get this error? How do I get around it? What I am trying to do is read a google sheet and use it in my function.
from google.oauth2 import service_account
credentials = service_account.Credentials.from_service_account_file(
SERVICE_ACCOUNT_FILE, scopes=SCOPES)
delegated_credentials = credentials.with_subject(EMAIL_FROM)
the google function logs give the following error
textPayload: "ModuleNotFoundError: No module named 'oauth2client'"
ImportError: file_cache is unavailable when using oauth2client >= 4.0.0 or google-auth
How do I resolve it?
You have to change your dependency oauth2client, it is deprecated as mentioned here
You can prefer google-auth==1.13.1 and it should work
So, your next question will be: why it works on my local environment? I think you have installed google-auth globally and your code take this dependency even if it's not in the requirements.txt. I recommend you to work with virtual environment and to install the dependencies only in this venv.
Hope this helps, by the way target audience is the url that you want to access.
import argparse
import google.auth
import google.auth.app_engine
import google.auth.compute_engine.credentials
import google.auth.iam
from google.auth.transport.requests import Request
import google.oauth2.credentials
from google.oauth2 import service_account
class AuthenticationConstants:
AUTHENTICATION_SCOPES_URL = 'https://www.googleapis.com/auth/cloud-platform'
OAUTH_TOKEN_URI = 'https://www.googleapis.com/oauth2/v4/token'
class JWT(object):
def __init__(self, service_account_key_path):
self.service_account_key_path = service_account_key_path
self.credentials = service_account.Credentials.from_service_account_file(
self.service_account_key_path)
self.scoped_credentials = self.credentials.with_scopes(
[AuthenticationConstants.OAUTH_TOKEN_URI])
def get_google_open_id_connect_token(self, target_audience):
signer_email = self.scoped_credentials.service_account_email
signer = self.scoped_credentials.signer
service_account_credentials = google.oauth2.service_account.Credentials(
signer,
signer_email,
token_uri=AuthenticationConstants.OAUTH_TOKEN_URI,
additional_claims={
'target_audience': target_audience
}
)
service_account_jwt = service_account_credentials._make_authorization_grant_assertion()
request = google.auth.transport.requests.Request()
body = {
'assertion': service_account_jwt,
'grant_type': google.oauth2._client._JWT_GRANT_TYPE,
}
token_response = google.oauth2._client._token_endpoint_request(
request, AuthenticationConstants.OAUTH_TOKEN_URI, body)
return token_response['id_token']
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("--service-account-path")
parser.add_argument("--target-audience")
result = parser.parse_args()
jwt = JWT(result.service_account_path)
print(jwt.get_google_open_id_connect_token(result.target_audience))
The following is the requirements.txt that I use:
google-api-python-client==1.7.11
Related
I deployed a model to the model registry on Vertex AI. I added an endpoint too, and I am able to make inferences. Below is the code that I wrote (using Python 3.9.12):
from google.cloud import aiplatform
from google.oauth2 import service_account
# settings is a Pydantic BaseSettings subclass object
credentials_json = json.loads(settings.GCP_VERTEX_SERVICE_ACC)
credentials = service_account.Credentials.from_service_account_info(
info=credentials_json
)
aiplatform.init(project=settings.GCLOUD_PROJECT_NUMBER,
location=settings.GCLOUD_LOCATION,
credentials=credentials)
endpoint = aiplatform.Endpoint(settings.GCLOUD_SBERT_ENDPOINT_ID)
...
async def do_inference(list_strs: List[str]):
result = endpoint.predict(instances=list_strs)
return result.predictions
Right now I'm not able to make asynchronous requests. Is there a way around this? For instance, would using the aiplatform_v1beta1.PredictionServiceAsyncClient library be a solution? Thanks in advance!
---- EDIT -----
Below is the piece of code that did it for me in case someone else is struggling with the same thing.
import asyncio
from google.cloud import aiplatform_v1beta1
from google.oauth2 import service_account
from google.protobuf import json_format
from google.protobuf.struct_pb2 import Value
# settings is a Pydantic BaseSettings subclass object
credentials_json = json.loads(settings.GCP_VERTEX_SERVICE_ACC)
credentials = service_account.Credentials.from_service_account_info(
info=credentials_json
)
client_options = {"api_endpoint": f"{settings.GCLOUD_LOCATION}-aiplatform.googleapis.com"}
client = aiplatform_v1beta1.PredictionServiceAsyncClient(credentials=credentials, client_options=client_options)
...
async def do_inference(list_strs: List[str]):
request = aiplatform_v1beta1.PredictRequest(endpoint=endpoint)
request.instances.extend(list_strs)
response = await client.predict(request)
predictions = response.predictions
return predictions
asyncio.get_event_loop().run_until_complete(do_inference())
This code owes a lot to #milad_raesi's answer!
I'm writing a program to backup my Google Photos library locally, and last week the code was working fine but in the last few days I've been repeatedly getting an error back from the API client saying that "photoslibrary v1" does not exist within Google's API library. The docs haven't changed at all - is there a problem on Google's end or is it me?
from googleapiclient import discovery
from httplib2 import Http
from oauth2client import file, client, tools
class Photos:
def __init__(self):
self.SCOPE = "https://www.googleapis.com/auth/photoslibrary"
self.CLIENT_SECRET = "client_id.json"
self.store = file.Storage("storage.json")
self.credentials = self.store.get()
if not self.credentials or self.credentials.invalid:
self.flow = client.flow_from_clientsecrets("client_id.json", self.SCOPE)
self.credentials = tools.run_flow(self.flow, self.store)
self.PHOTOS = discovery.build("photoslibrary", "v1", http=self.credentials.authorize(Http()))
photos = Photos()
googleapiclient.errors.UnknownApiNameOrVersion: name: photoslibrary version: v1
Try adding the parameter static_discovery=False
self.PHOTOS = discovery.build("photoslibrary", "v1", http=self.credentials.authorize(Http()),static_discovery=False)
I am trying to add a reply_url programmatically to an Azure app registration, but I receive an azure.graphrbac.models.graph_error_py3.GraphErrorException: Specified HTTP method is not allowed for the request target.
It fails when I try to update an existing application with new reply_urls.
SDK I am using is: azure-graphrbac==0.61.1
My code:
from azure.common.credentials import ServicePrincipalCredentials
from azure.graphrbac import GraphRbacManagementClient
from azure.graphrbac.models import ApplicationUpdateParameters
class GraphClient:
def __init__(self, client_id, client_secret, tenant_id, object_id):
self._credentials = ServicePrincipalCredentials(
client_id=client_id,
secret=client_secret,
tenant=tenant_id,
resource="https://graph.windows.net"
)
self._graph_client = GraphRbacManagementClient(
credentials=self._credentials,
tenant_id=tenant_id
)
self._object_id = object_id
self._application = self._graph_client.applications.get(self._object_id)
def get_reply_urls(self) -> List[str]:
return self._application.reply_urls
def add_reply_url(self, reply_url) -> None:
reply_urls: list = self.get_reply_urls()
self._graph_client.applications.patch(
self._object_id,
ApplicationUpdateParameters(
reply_urls=[
*reply_urls,
reply_url]
)
)
Could not reproduce your issue, use the same version of azure-graphrbac, I test your code on my side, it works fine.
testclient = GraphClient(client_id = "xxxxx",client_secret = "xxxxx", tenant_id = "xxxxx", object_id = "xxxxx")
testclient.add_reply_url(reply_url = "http://localhost:8085")
Check in the portal:
Also, I test the sdk directly, both work.
from azure.common.credentials import ServicePrincipalCredentials
from azure.graphrbac import GraphRbacManagementClient
from azure.graphrbac.models import ApplicationUpdateParameters
_credentials = ServicePrincipalCredentials(
client_id="xxxxx",
secret="xxxxx",
tenant="xxxxx",
resource="https://graph.windows.net"
)
_graph_client = GraphRbacManagementClient(
credentials=_credentials,
tenant_id="xxxxx"
)
app = _graph_client.applications.patch(
application_object_id = "xxxxx",
parameters = ApplicationUpdateParameters(reply_urls = ["http://localhost:8080","http://localhost:8081"])
)
Update call looks good however it depends on a legacy API (AAD Graph) and tools. It's strongly recommended to move to MS Graph which supports almost all Azure AD Graph operations and will fully support them all in a future. Applications being one of them.
You can use Requests-OAuthlib or Microsoft Graph Python Client Library for this.
import json
from httplib2 import Http
from oauth2client.client import SignedJwtAssertionCredentials
from googleapiclient.discovery import build
json_file = 'my.json'
client_email = json.loads(open(json_file).read())['client_email']
private_key = json.loads(open(json_file).read())['private_key']
cloud_storage_bucket = 'my_bucket'
report_to_download = 'sales/salesreport_201907.zip'
credentials = SignedJwtAssertionCredentials(client_email, private_key,['https://www.googleapis.com/auth/devstorage.read_only','https://www.googleapis.com/auth/devstorage.read_write','https://www.googleapis.com/auth/devstorage.full_control'])
storage = build('storage', 'v1', http=credentials.authorize(Http()))
object_metadata = storage.objects().get(bucket = cloud_storage_bucket, object = report_to_download).execute()
object_content = storage.objects().get_media(bucket = cloud_storage_bucket, object = report_to_download).execute()
I can read the object_metadata, but when requesting object content I get:
googleapiclient.errors.HttpError: <HttpError 403 when requesting https://www.googleapis.com/storage/v1/b/my_bucket/o/sales%2Fsalesreport_201907.zip?alt=media returned "Forbidden">
As u can see I added the three scopes in the credentials sections just to ensure to have all the permissions. Despite that I still getting the error.
I am using Python 2.7.
Edit:
I tried to download the file using gsutil and it works. So it isn't related with permissions.
Edit 2:
Changed the code to use a non deprecated library.
from logs import logger
import google.auth
import google.auth.transport.requests as tr_requests
from google.resumable_media.requests import Download
import os
os.environ["GOOGLE_APPLICATION_CREDENTIALS"]="my.json"
ro_scope=u'https://www.googleapis.com/auth/devstorage.full_control'
credentials,_ = google.auth.default(scopes=(ro_scope,))
transport = tr_requests.AuthorizedSession(credentials)
cloud_storage_bucket = 'pubsite_prod_rev_xxxx'
report_to_download = 'sales/salesreport_201907.zip'
#option 1
media_url='https://www.googleapis.com/download/storage/v1/b/pubsite_prod_rev_xxxx/o/sales%2Fsalesreport_201907.zip?generation=1234&alt=media'
#option 2
media_url='https://www.googleapis.com/storage/v1/b/pubsite_prod_rev_xxxx/o/sales%2Fsalesreport_201907.zip'
download = Download(media_url)
response = download.consume(transport)
print download.finished
When running the code with option 1 url the status is 200 but the response.content is just the metadata, has happened before with the get method.
When running the option 2, like get_media method, the error stills 403.
File "/anaconda2/envs/enviro27/lib/python2.7/site-packages/google/resumable_media/_helpers.py", line 93, in require_status_code
status_code, u'Expected one of', *status_codes)
google.resumable_media.common.InvalidResponse: (u'Request failed with status code', 403, u'Expected one of', 200, 206)
The doc of this code.
The solution has been to discover the true admin account.
Despite Google Play shows contact information to one email (the one that we believe to be the admin), the real admin is anothe account. After find it, we applied all the process explained here.
And with this code we are able to access the content:
from logs import logger
import google.auth
import google.auth.transport.requests as tr_requests
from google.resumable_media.requests import Download
import os
os.environ["GOOGLE_APPLICATION_CREDENTIALS"]="my.json"
ro_scope=u'https://www.googleapis.com/auth/devstorage.full_control'
credentials,_ = google.auth.default(scopes=(ro_scope,))
transport = tr_requests.AuthorizedSession(credentials)
cloud_storage_bucket = 'pubsite_prod_rev_xxxx'
report_to_download = 'sales/salesreport_201907.zip'
#option 1
media_url='https://www.googleapis.com/download/storage/v1/b/pubsite_prod_rev_xxxx/o/sales%2Fsalesreport_201907.zip?generation=1234&alt=media'
#option 2
media_url='https://www.googleapis.com/storage/v1/b/pubsite_prod_rev_xxxx/o/sales%2Fsalesreport_201907.zip'
download = Download(media_url)
response = download.consume(transport)
print download.finished
Unable to get all results of mediaItems.search:
photos = google.get_service(credentials, 'photoslibrary', 'v1')
request = photos.albums().list(pageSize=50)
while request is not None:
result = request.execute()
for album in result['albums']:
request2 = photos.mediaItems().search(body={'albumId': album['id']})
while request2 is not None:
result2 = request2.execute()
request2 = photos.mediaItems().search_next(request2, result2)
print('nextpageToken' in result2, request2)
request = photos.albums().list_next(request, result)
Running this fails on the first search_next() call with
[...]
File "/usr/local/lib/python3.7/dist-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper
return wrapped(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/googleapiclient/http.py", line 851, in execute
raise HttpError(resp, content, uri=self.uri)
googleapiclient.errors.HttpError: <HttpError 400 when requesting https://photoslibrary.googleapis.com/v1/mediaItems:search?alt=json returned "Invalid JSON payload received. Unexpected end of string. Expected an object key or }.
That library isn't really supported it seems, so that might be the problem, or am I missing something here?
The google-api-python-client is generic client for all the Google's discovery based APIs and as such it supports all the API based on this protocol including the Photos one.
The correct way to use the services is to invoke the build method and after that use the available methods of the services.
Beside this you always want to use list_next as search_next does not exists.
Here is an example of the photos API working on my laptop (python 3.6)
import os
import pickle
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
SCOPES = ['https://www.googleapis.com/auth/photoslibrary.readonly', ]
# we check if we save the credentials in the past and we reuse them
if not os.path.exists('credentials.dat'):
# no credentials found, we run the standard auth flow
flow = InstalledAppFlow.from_client_secrets_file('client_id.json', SCOPES)
credentials = flow.run_local_server()
with open('credentials.dat', 'wb') as credentials_dat:
pickle.dump(credentials, credentials_dat)
else:
with open('credentials.dat', 'rb') as credentials_dat:
credentials = pickle.load(credentials_dat)
if credentials.expired:
credentials.refresh(Request())
photos_sdk = build('photoslibrary', 'v1', credentials=credentials)
# photos API
photos_albums_api = photos_sdk.albums()
photos_mediaitems_api = photos_sdk.mediaItems()
albums_list_params = {
'pageSize': 50,
}
# first request
albums_list_req = photos_albums_api.list(**albums_list_params)
while albums_list_req is not None:
photos_albums_list = albums_list_req.execute()
# print(photos_albums_list)
for album in photos_albums_list['albums']:
print(album['title'])
mediaitems_search_req = photos_mediaitems_api.search(body={'albumId': album['id']})
while mediaitems_search_req is not None:
mediaitems_search = mediaitems_search_req.execute()
print(mediaitems_search)
# mediaItems pagination management
mediaitems_search_req = photos_mediaitems_api.list_next(mediaitems_search_req, mediaitems_search)
# albums pagination handling
albums_list_req = photos_albums_api.list_next(albums_list_req, photos_albums_list)
If there are more results than the specified pageSize, the API returns a pageToken, you should use to request the next part. See the example here Access Google Photo API with Python using google-api-python-client