I've recently open a question about the GA management API where I explained that the exemple script here didn't worked for me. So looking at the Google Analytics Git Hub, I found a totaly different exemple which doesn't work either but I have a different error :
The client secrets were invalid:
Missing property "client_secret" in a client type of "installed".
WARNING: Please configure OAuth 2.0
To make this sample run you will need to populate the client_secrets.json file
found at:
client_secrets.json
with information from the APIs Console <https://code.google.com/apis/console>.
However I did all this perfectly well, I have my client_secrets.json file. what are the differences between client_secrets authentification and p12 authentification ? Is it possible that the Management API work only with client_secrets authentification ? Does someone have a working code example ?
Thanks.
Here is the script :
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright 2014 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Reference command-line example for Google Analytics Management API v3.
This application demonstrates how to use the python client library to access
all the pieces of data returned by the Google Analytics Management API v3.
The application manages autorization by saving an OAuth2.0 token in a local
file and reusing the token for subsequent requests. It then traverses the
Google Analytics Management hiearchy. It first retrieves and prints all the
authorized user's accounts, next it prints all the web properties for the
first account, then all the profiles for the first web property and finally
all the goals for the first profile. The sample then prints all the
user's advanced segments.
Before You Begin:
Update the client_secrets.json file
You must update the clients_secrets.json file with a client id, client
secret, and the redirect uri. You get these values by creating a new project
in the Google APIs console and registering for OAuth2.0 for installed
applications: https://code.google.com/apis/console
Learn more about registering your analytics application here:
https://developers.google.com/analytics/devguides/config/mgmt/v3/mgmtAuthorization
Sample Usage:
$ python management_v3_reference.py
Also you can also get help on all the command-line flags the program
understands by running:
$ python management_v3_reference.py --help
"""
from __future__ import print_function
__author__ = 'api.nickm#gmail.com (Nick Mihailovski)'
import argparse
import sys
from googleapiclient.errors import HttpError
from googleapiclient import sample_tools
from oauth2client.client import AccessTokenRefreshError
def main(argv):
# Authenticate and construct service.
service, flags = sample_tools.init(
argv, 'analytics', 'v3', __doc__, __file__,
scope='https://www.googleapis.com/auth/analytics.readonly')
# Traverse the Management hiearchy and print results or handle errors.
try:
traverse_hiearchy(service)
except TypeError as error:
# Handle errors in constructing a query.
print(('There was an error in constructing your query : %s' % error))
except HttpError as error:
# Handle API errors.
print(('Arg, there was an API error : %s : %s' %
(error.resp.status, error._get_reason())))
except AccessTokenRefreshError:
print ('The credentials have been revoked or expired, please re-run'
'the application to re-authorize')
def traverse_hiearchy(service):
"""Traverses the management API hiearchy and prints results.
This retrieves and prints the authorized user's accounts. It then
retrieves and prints all the web properties for the first account,
retrieves and prints all the profiles for the first web property,
and retrieves and prints all the goals for the first profile.
Args:
service: The service object built by the Google API Python client library.
Raises:
HttpError: If an error occured when accessing the API.
AccessTokenRefreshError: If the current token was invalid.
"""
accounts = service.management().accounts().list().execute()
print_accounts(accounts)
if accounts.get('items'):
firstAccountId = accounts.get('items')[0].get('id')
webproperties = service.management().webproperties().list(
accountId=firstAccountId).execute()
print_webproperties(webproperties)
if webproperties.get('items'):
firstWebpropertyId = webproperties.get('items')[0].get('id')
profiles = service.management().profiles().list(
accountId=firstAccountId,
webPropertyId=firstWebpropertyId).execute()
print_profiles(profiles)
if profiles.get('items'):
firstProfileId = profiles.get('items')[0].get('id')
goals = service.management().goals().list(
accountId=firstAccountId,
webPropertyId=firstWebpropertyId,
profileId=firstProfileId).execute()
print_goals(goals)
print_segments(service.management().segments().list().execute())
def print_accounts(accounts_response):
"""Prints all the account info in the Accounts Collection.
Args:
accounts_response: The response object returned from querying the Accounts
collection.
"""
print('------ Account Collection -------')
print_pagination_info(accounts_response)
print()
for account in accounts_response.get('items', []):
print('Account ID = %s' % account.get('id'))
print('Kind = %s' % account.get('kind'))
print('Self Link = %s' % account.get('selfLink'))
print('Account Name = %s' % account.get('name'))
print('Created = %s' % account.get('created'))
print('Updated = %s' % account.get('updated'))
child_link = account.get('childLink')
print('Child link href = %s' % child_link.get('href'))
print('Child link type = %s' % child_link.get('type'))
print()
if not accounts_response.get('items'):
print('No accounts found.\n')
def print_webproperties(webproperties_response):
"""Prints all the web property info in the WebProperties collection.
Args:
webproperties_response: The response object returned from querying the
Webproperties collection.
"""
print('------ Web Properties Collection -------')
print_pagination_info(webproperties_response)
print()
for webproperty in webproperties_response.get('items', []):
print('Kind = %s' % webproperty.get('kind'))
print('Account ID = %s' % webproperty.get('accountId'))
print('Web Property ID = %s' % webproperty.get('id'))
print(('Internal Web Property ID = %s' %
webproperty.get('internalWebPropertyId')))
print('Website URL = %s' % webproperty.get('websiteUrl'))
print('Created = %s' % webproperty.get('created'))
print('Updated = %s' % webproperty.get('updated'))
print('Self Link = %s' % webproperty.get('selfLink'))
parent_link = webproperty.get('parentLink')
print('Parent link href = %s' % parent_link.get('href'))
print('Parent link type = %s' % parent_link.get('type'))
child_link = webproperty.get('childLink')
print('Child link href = %s' % child_link.get('href'))
print('Child link type = %s' % child_link.get('type'))
print()
if not webproperties_response.get('items'):
print('No webproperties found.\n')
def print_profiles(profiles_response):
"""Prints all the profile info in the Profiles Collection.
Args:
profiles_response: The response object returned from querying the
Profiles collection.
"""
print('------ Profiles Collection -------')
print_pagination_info(profiles_response)
print()
for profile in profiles_response.get('items', []):
print('Kind = %s' % profile.get('kind'))
print('Account ID = %s' % profile.get('accountId'))
print('Web Property ID = %s' % profile.get('webPropertyId'))
print(('Internal Web Property ID = %s' %
profile.get('internalWebPropertyId')))
print('Profile ID = %s' % profile.get('id'))
print('Profile Name = %s' % profile.get('name'))
print('Currency = %s' % profile.get('currency'))
print('Timezone = %s' % profile.get('timezone'))
print('Default Page = %s' % profile.get('defaultPage'))
print(('Exclude Query Parameters = %s' %
profile.get('excludeQueryParameters')))
print(('Site Search Category Parameters = %s' %
profile.get('siteSearchCategoryParameters')))
print(('Site Search Query Parameters = %s' %
profile.get('siteSearchQueryParameters')))
print('Created = %s' % profile.get('created'))
print('Updated = %s' % profile.get('updated'))
print('Self Link = %s' % profile.get('selfLink'))
parent_link = profile.get('parentLink')
print('Parent link href = %s' % parent_link.get('href'))
print('Parent link type = %s' % parent_link.get('type'))
child_link = profile.get('childLink')
print('Child link href = %s' % child_link.get('href'))
print('Child link type = %s' % child_link.get('type'))
print()
if not profiles_response.get('items'):
print('No profiles found.\n')
def print_goals(goals_response):
"""Prints all the goal info in the Goals collection.
Args:
goals_response: The response object returned from querying the Goals
collection
"""
print('------ Goals Collection -------')
print_pagination_info(goals_response)
print()
for goal in goals_response.get('items', []):
print('Goal ID = %s' % goal.get('id'))
print('Kind = %s' % goal.get('kind'))
print('Self Link = %s' % goal.get('selfLink'))
print('Account ID = %s' % goal.get('accountId'))
print('Web Property ID = %s' % goal.get('webPropertyId'))
print(('Internal Web Property ID = %s' %
goal.get('internalWebPropertyId')))
print('Profile ID = %s' % goal.get('profileId'))
print('Goal Name = %s' % goal.get('name'))
print('Goal Value = %s' % goal.get('value'))
print('Goal Active = %s' % goal.get('active'))
print('Goal Type = %s' % goal.get('type'))
print('Created = %s' % goal.get('created'))
print('Updated = %s' % goal.get('updated'))
parent_link = goal.get('parentLink')
print('Parent link href = %s' % parent_link.get('href'))
print('Parent link type = %s' % parent_link.get('type'))
# Print the goal details depending on the type of goal.
if goal.get('urlDestinationDetails'):
print_url_destination_goal_details(
goal.get('urlDestinationDetails'))
elif goal.get('visitTimeOnSiteDetails'):
print_visit_time_on_site_goal_details(
goal.get('visitTimeOnSiteDetails'))
elif goal.get('visitNumPagesDetails'):
print_visit_num_pages_goal_details(
goal.get('visitNumPagesDetails'))
elif goal.get('eventDetails'):
print_event_goal_details(goal.get('eventDetails'))
print()
if not goals_response.get('items'):
print('No goals found.\n')
def print_url_destination_goal_details(goal_details):
"""Prints all the URL Destination goal type info.
Args:
goal_details: The details portion of the goal response.
"""
print('------ Url Destination Goal -------')
print('Goal URL = %s' % goal_details.get('url'))
print('Case Sensitive = %s' % goal_details.get('caseSensitive'))
print('Match Type = %s' % goal_details.get('matchType'))
print('First Step Required = %s' % goal_details.get('firstStepRequired'))
print('------ Url Destination Goal Steps -------')
for goal_step in goal_details.get('steps', []):
print('Step Number = %s' % goal_step.get('number'))
print('Step Name = %s' % goal_step.get('name'))
print('Step URL = %s' % goal_step.get('url'))
if not goal_details.get('steps'):
print('No Steps Configured')
def print_visit_time_on_site_goal_details(goal_details):
"""Prints all the Visit Time On Site goal type info.
Args:
goal_details: The details portion of the goal response.
"""
print('------ Visit Time On Site Goal -------')
print('Comparison Type = %s' % goal_details.get('comparisonType'))
print('comparison Value = %s' % goal_details.get('comparisonValue'))
def print_visit_num_pages_goal_details(goal_details):
"""Prints all the Visit Num Pages goal type info.
Args:
goal_details: The details portion of the goal response.
"""
print('------ Visit Num Pages Goal -------')
print('Comparison Type = %s' % goal_details.get('comparisonType'))
print('comparison Value = %s' % goal_details.get('comparisonValue'))
def print_event_goal_details(goal_details):
"""Prints all the Event goal type info.
Args:
goal_details: The details portion of the goal response.
"""
print('------ Event Goal -------')
print('Use Event Value = %s' % goal_details.get('useEventValue'))
for event_condition in goal_details.get('eventConditions', []):
event_type = event_condition.get('type')
print('Type = %s' % event_type)
if event_type in ('CATEGORY', 'ACTION', 'LABEL'):
print('Match Type = %s' % event_condition.get('matchType'))
print('Expression = %s' % event_condition.get('expression'))
else: # VALUE type.
print('Comparison Type = %s' % event_condition.get('comparisonType'))
print('Comparison Value = %s' % event_condition.get('comparisonValue'))
def print_segments(segments_response):
"""Prints all the segment info in the Segments collection.
Args:
segments_response: The response object returned from querying the
Segments collection.
"""
print('------ Segments Collection -------')
print_pagination_info(segments_response)
print()
for segment in segments_response.get('items', []):
print('Segment ID = %s' % segment.get('id'))
print('Kind = %s' % segment.get('kind'))
print('Self Link = %s' % segment.get('selfLink'))
print('Name = %s' % segment.get('name'))
print('Definition = %s' % segment.get('definition'))
print('Created = %s' % segment.get('created'))
print('Updated = %s' % segment.get('updated'))
print()
def print_pagination_info(management_response):
"""Prints common pagination details.
Args:
management_response: The common reponse object for each collection in the
Management API.
"""
print('Items per page = %s' % management_response.get('itemsPerPage'))
print('Total Results = %s' % management_response.get('totalResults'))
print('Start Index = %s' % management_response.get('startIndex'))
# These only have values if other result pages exist.
if management_response.get('previousLink'):
print('Previous Link = %s' % management_response.get('previousLink'))
if management_response.get('nextLink'):
print('Next Link = %s' % management_response.get('nextLink'))
if __name__ == '__main__':
main(sys.argv)
client_secrets.json is Oauth2 authentication that being a user is first prompted for access to their account.
P12 file is service account authentication. Service accounts are like dummy users they are pre authorized which means there will be no prompt for user consent. To pre authorize a service account to Google Analytics the service account email address must be added as a user at the ACCOUNT level in the admin section of the google analytics website. Yes service accounts work with the management API.
Note: Oauth2 credentials and service account credentials are created on the google developer console when registering your application. It is now possible to get a .json file for service accounts instead of only the p12. However the code used to authenticate a service account and Oauth2 are different and these two .json files can not be interchanged. (hope that makes sense)
Further reading:
My tutorial on Oauth2 Google Developer Console Oauth2 credentials
My tutorial on service accounts Google Developer console service account
Related
Im trying to stop my RDS cluster using Lambda and Im using the python code shown below to target the clusters tags,
However when trying to create a CloudFormation stack, AWS is telling me that the YAML is not well formed.
For example it tells that Key = 'True' is not well formed.
If I remove that bit of code it then tells me the next bit of code, client = boto3.client('rds') is also not well formed.
Ive put this code into an online python validator and it didnt report any issues.
Can anyone help with this? Thanks
Tag = 'AutoPower'
Key = 'True'
client = boto3.client('rds')
response = client.describe_db_cluster()
for resp in response['DBCluster']:
db_cluster_arn = resp['DBClusterArn']
response = client.list_tags_for_resource(ResourceName=db_cluster_arn)
for tags in response['TagList']:
if tags['Key'] == str(Key) and tags['Value'] == str(Tag):
status = resp['DBClusterStatus']
ClusterID = resp['DBClusterIdentifier']
print(InstanceID)
if status == 'available':
print("shutting down %s " % ClusterID)
client.stop_db_cluster(DBClusterIdentifier=ClusterID)
# print ("Do something with it : %s" % db_instance_arn)
elif status == 'stopped':
print("starting up %s " % ClusterID)
client.start_db_cluster(DBClusterIdentifier=ClusterID)
else:
print("The database is " + status + " status!")
You are most likely having issues due to indentation issues between CloudFormation YAML and the inline function code. Here is an example of CloudFormation YAML that would use your code inline to create a function:
Resources:
YourFunction:
Type: AWS::Lambda::Function
Properties:
Code:
ZipFile: |
Tag = 'AutoPower'
Key = 'True'
client = boto3.client('rds')
response = client.describe_db_cluster()
for resp in response['DBCluster']:
db_cluster_arn = resp['DBClusterArn']
response = client.list_tags_for_resource(ResourceName=db_cluster_arn)
for tags in response['TagList']:
if tags['Key'] == str(Key) and tags['Value'] == str(Tag):
status = resp['DBClusterStatus']
ClusterID = resp['DBClusterIdentifier']
print(InstanceID)
if status == 'available':
print("shutting down %s " % ClusterID)
client.stop_db_cluster(DBClusterIdentifier=ClusterID)
# print ("Do something with it : %s" % db_instance_arn)
elif status == 'stopped':
print("starting up %s " % ClusterID)
client.start_db_cluster(DBClusterIdentifier=ClusterID)
else:
print("The database is " + status + " status!")
Handler: index.lambda_handler
Role: !Sub "arn:aws:iam::${AWS::AccountId}:role/YourRoleNameHere"
Runtime: python3.9
Description: This is an example of a function in CloudFormation
FunctionName: Your_Function_Name
MemorySize: 128
Timeout: 180
I'm using python-acme to write a small script that does takes in a domain and does one of two things:
Using the DNS01 challenge, if the challenge doesn't pass, return the DNS entry that needs to be added to the domain.
Using the DNS01 challenge, if the challenge passes, return the certificate information.
I'm running into issues however, and the documentation is just terrible :P I'm trying to find a way to "reuse" the order/challenge. Basically, any time I run the script so far, the validation token for the DNS entry changes.
print("Validating Challenge...")
response, validation = challenge.response_and_validation(client_acme.net.key)
print("response %s" % response.to_partial_json())
print("validation %s" % validation)
print("-> Validation Domain: %s" % challenge.chall.validation_domain_name(domain))
print("-> Validation Value: %s" % challenge.chall.validation(client_acme.net.key))
# TODO - We are here, gotta actually get info on the DNS challange and attempt to validate.
print("Validation Completed!")
Things I've tried:
Register the account with the same RSA key. Runs into validation errors when registering.
Running new_order with the same CSR. Still returns a different key.
Full Code (In It's Prototype Glory)
import json
import josepy as jose
import os
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.serialization import load_pem_private_key
from acme import client
from acme import messages
from acme import challenges
from acme import crypto_util
import OpenSSL
from boto.s3.connection import S3Connection
from boto.s3.key import Key
"""
Generates an SSL certificate through Let's Encrypt provided a domain. If the
domain requires a DNS challenge to be passed, that information is passed back
to the user. Otherwise, the SSL certificate is generated and returned back to
the user.
Documentation for generating SSL via this method:
http://www.gilesthomas.com/2018/11/python-code-to-generate-lets-encrypt-certificates/
ACME Documentation:
https://kite.com/python/docs/acme.client.ClientV2.new_order
"""
print('Loading function')
DEBUG = True
DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory'
KEY_SIZE = 2048
CERT_PKEY_BITS = 2048
USER_AGENT = 'python-acme-example'
EMAIL_ADDRESS = 'REDACTED'
S3_BUCKET = 'REDACTED'
S3_KEY = 'REDACTED'
S3_SECRET = 'REDACTED'
# TODO - Load These From Event
PASSWORD = "swordfish"
SALT = "yourAppName"
def new_csr_comp(domain_name, pkey_pem=None):
"""Create certificate signing request."""
if pkey_pem is None:
# Create private key.
pkey = OpenSSL.crypto.PKey()
pkey.generate_key(OpenSSL.crypto.TYPE_RSA, CERT_PKEY_BITS)
pkey_pem = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM,
pkey)
csr_pem = crypto_util.make_csr(pkey_pem, [domain_name])
return pkey_pem, csr_pem
def save_key(pk, filename):
pem = pk.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption()
)
return pem
def lambda_handler(event, context):
print("Formatting Domain...")
domain = event['domain']
domain = domain.lower()
print("Formatted! Domain is: %s" % domain)
print("Generating User Key...")
conn = S3Connection(S3_KEY, S3_SECRET)
bucket = conn.get_bucket(S3_BUCKET)
key_name = "%s.key" % domain
existing_account = False
print("-> Looking For Existing Key.")
rsa_key = bucket.get_key(key_name)
if rsa_key is None or DEBUG:
print("-> Key Not Found. Creating New One.")
rsa_key = rsa.generate_private_key(
public_exponent=65537,
key_size=KEY_SIZE,
backend=default_backend()
)
pem = save_key(rsa_key, key_name)
print(key_name)
k = Key(bucket)
k.key = key_name
k.set_contents_from_string(pem)
else:
print("-> Key File Found.")
existing_account = True
rsa_key = rsa_key.get_contents_as_string()
rsa_key = load_pem_private_key(rsa_key, password=None, backend=default_backend())
print(rsa_key)
print("-> Converted File To Usable Format.")
acc_key = jose.JWKRSA(
key=rsa_key
)
print("Generated!")
print("Registering With Let's Encrypt...")
print("-> Connecting to Let's Encrypt on {}".format(DIRECTORY_URL))
net = client.ClientNetwork(acc_key, user_agent=USER_AGENT)
directory = messages.Directory.from_json(net.get(DIRECTORY_URL).json())
client_acme = client.ClientV2(directory, net=net)
print("-> Registering")
email = (EMAIL_ADDRESS)
regr = None
# TODO - Use Existing Account
# if existing_account:
# regr = messages.NewRegistration(key=acc_key.public_key(), only_return_existing=True)
# else:
account_created = messages.NewRegistration.from_data(email=email, terms_of_service_agreed=True)
regr = client_acme.new_account(account_created)
print("Registered!")
print("Creating CSR...")
temp_pkey_pem, temp_csr_pem = new_csr_comp(domain)
key_name = "%s.pkey_pem" % domain
pkey_pem = bucket.get_key(key_name)
if pkey_pem is None:
print("-> Creating New PKEY")
k = Key(bucket)
k.key = key_name
k.set_contents_from_string(temp_pkey_pem)
pkey_pem = temp_pkey_pem
else:
print("-> Using Existing PKEY")
pkey_pem = pkey_pem.get_contents_as_string()
key_name = "%s.csr_pem" % domain
csr_pem = bucket.get_key(key_name)
if csr_pem is None:
print("-> Creating New CSR")
k = Key(bucket)
k.key = key_name
k.set_contents_from_string(temp_csr_pem)
csr_pem = temp_csr_pem
else:
print("-> Using Existing CSR")
csr_pem = csr_pem.get_contents_as_string()
print("Created!")
print("Requesting Challenges...")
orderr = client_acme.new_order(csr_pem)
print("Requested!")
print("Selecting DNS Challenge...")
challenge = None
authz_list = orderr.authorizations
for authz in authz_list:
for i in authz.body.challenges:
if isinstance(i.chall, challenges.DNS01):
challenge = i
else:
print("-> Other challenge found: %s" % i.chall)
if challenge is None:
raise Exception("Could not find a DNS challenge!")
print("Selected!")
print("Validating Challenge...")
response, validation = challenge.response_and_validation(client_acme.net.key)
print("response %s" % response.to_partial_json())
print("validation %s" % validation)
print("-> Validation Domain: %s" % challenge.chall.validation_domain_name(domain))
print("-> Validation Value: %s" % challenge.chall.validation(client_acme.net.key))
# TODO - We are here, gotta actually get info on the DNS challange and attempt to validate.
print("Validation Completed!")
print("Starting Challenge...")
client_acme.answer_challenge(challenge, response)
finalized_orderr = client_acme.poll_and_finalize(orderr)
fullchain_pem = finalized_orderr.fullchain_pem
print("-> PEM: %s" % fullchain_pem)
print("Challenge Completed!")
# TODO - We need to return the DNS challenge if it hasn't been completed yet.
return "done"
#raise Exception('Something went wrong')
Bryant
print("Starting Challenge...")
client_acme.answer_challenge(challenge, response)
finalized_orderr = client_acme.poll_and_finalize(orderr)
fullchain_pem = finalized_orderr.fullchain_pem
print("-> PEM: %s" % fullchain_pem)
print("Challenge Completed!")`enter code here`
Each time you run this code block, it will send a challenge to LetsEncrypt Server. If this challenge success it will return your certs and everything ok, but if challenge invalid LetsEncrypt Server will change your Key Authoriztion and TXT value.
Hope, its help.
I have a python 3.6 AWS lambda function "ftp_sender" which is triggered by ObjectCreated event from S3. Then it downloads the file and sends it to the SFTP server.
And in CloudWatch logs I'm constantly seeing the messages like that:
[ERROR] 2018-07-26T12:30:21.543Z a56f9678-90c9-11e8-bf10-ddb8557d0ff0 Socket exception: Operation not permitted (1)
This messages appear in random positions even before the function started working, for example:
def ftp_sender(event, context):
# The error can be triggered here
print(event)
This seems not treated as a real error, but maybe someone knows what it is about?
[EDIT]
It looks like the issue caused by paramiko library. I have the following class inside the function:
class SftpSender:
def __init__(self, hostname, username, the_password, sftp_port=22, working_directory=''):
self.hostname = hostname
self.username = username
print('SFTP port: %s' % sftp_port)
self.transport = paramiko.Transport((hostname, sftp_port))
self.transport.connect(username=username, password=the_password)
self.sftp = paramiko.SFTPClient.from_transport(self.transport)
# self.transport.set_missing_host_key_policy(paramiko.WarningPolicy())
self.folder = working_directory
def send_file(self, filename, retries=4, timeout_between_retries=10, verb=True, the_lambda_mode=False):
retry_number = 0
sending_result = 'Not run'
if the_lambda_mode:
os.chdir("/tmp/")
if not os.path.isfile(filename):
return "Error: File %s is not found" % filename
while retry_number < retries:
if verb:
print('Sending %s to %s#%s (to folder "%s") retry %s of %s\n' % (filename, self.username, self.hostname, self.folder, retry_number, retries))
try:
destination_name = self.folder + '/' + filename
sending_result = self.sftp.put(filename, destination_name)
return 'OK'
except Exception as e:
print('Error: The following exception occured: "%s", result: "%s"' % (e, sending_result))
sleep(timeout_between_retries)
retry_number += 1
return 'Error: failed to send file %s (%s)' % (filename, e)
I'm using Raspberry Pi 3 & Python 2.7.9.
I'm trying to using youtube api. here's the code I use:
#!/usr/bin/python
from apiclient.discovery import build
from apiclient.errors import HttpError
from oauth2client.tools import argparser
import json
import urllib
# Set DEVELOPER_KEY to the API key value from the APIs & auth > Registered apps
# tab of
# https://cloud.google.com/console
# Please ensure that you have enabled the YouTube Data API for your project.
DEVELOPER_KEY = "REPLACE_ME" #I did replace api key
YOUTUBE_API_SERVICE_NAME = "youtube"
YOUTUBE_API_VERSION = "v3"
FREEBASE_SEARCH_URL = "https://www.googleapis.com/freebase/v1/search?%s"
def get_topic_id(options):
# Retrieve a list of Freebase topics associated with the provided query term.
freebase_params = dict(query=options.query, key=DEVELOPER_KEY)
freebase_url = FREEBASE_SEARCH_URL % urllib.urlencode(freebase_params)
freebase_response = json.loads(urllib.urlopen(freebase_url).read())
if len(freebase_response["result"]) == 0:
exit("No matching terms were found in Freebase.")
# Display the list of matching Freebase topics.
mids = []
index = 1
print "The following topics were found:"
for result in freebase_response["result"]:
mids.append(result["mid"])
print " %2d. %s (%s)" % (index, result.get("name", "Unknown"),
result.get("notable", {}).get("name", "Unknown"))
index += 1
# Display a prompt for the user to select a topic and return the topic ID
# of the selected topic.
mid = None
while mid is None:
index = raw_input("Enter a topic number to find related YouTube %ss: " %
options.type)
try:
mid = mids[int(index) - 1]
except ValueError:
pass
return mid
def youtube_search(mid, options):
youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION,
developerKey=DEVELOPER_KEY)
# Call the search.list method to retrieve results associated with the
# specified Freebase topic.
search_response = youtube.search().list(
topicId=mid,
type=options.type,
part="id,snippet",
maxResults=options.max_results
).execute()
# Print the title and ID of each matching resource.
for search_result in search_response.get("items", []):
if search_result["id"]["kind"] == "youtube#video":
print "%s (%s)" % (search_result["snippet"]["title"],
search_result["id"]["videoId"])
elif search_result["id"]["kind"] == "youtube#channel":
print "%s (%s)" % (search_result["snippet"]["title"],
search_result["id"]["channelId"])
elif search_result["id"]["kind"] == "youtube#playlist":
print "%s (%s)" % (search_result["snippet"]["title"],
search_result["id"]["playlistId"])
if __name__ == "__main__":
argparser.add_argument("--query", help="Freebase search term", default="Google")
argparser.add_argument("--max-results", help="Max YouTube results",
default=25)
argparser.add_argument("--type",
help="YouTube result type: video, playlist, or channel", default="channel")
args = argparser.parse_args()
mid = get_topic_id(args)
try:
youtube_search(mid, args)
except HttpError, e:
print "An HTTP error %d occurred:\n%s" % (e.resp.status, e.content)
And I got error:
ValueError: No JSON object could be decoded
I really don't know how to handle this... anyone how to fix this?
I think it's related with OAuth 2.0 json file, but not sure... and I don't know how to use it
I am trying to get tweets related to the keyword in the code But at the python shell there is nothing its just curson only No traceback nothing.The code is here
import time
import pycurl
import urllib
import json
import oauth2 as oauth
API_ENDPOINT_URL = 'https://stream.twitter.com/1.1/statuses/filter.json'
USER_AGENT = 'TwitterStream 1.0' # This can be anything really
# You need to replace these with your own values
OAUTH_KEYS = {'consumer_key': 'ABC',
'consumer_secret': 'ABC',
'access_token_key': 'ABC',
'access_token_secret': 'ABC'}
# These values are posted when setting up the connection
POST_PARAMS = {'include_entities': 0,
'stall_warning': 'true',
'track': 'iphone,ipad,ipod'}
class TwitterStream:
def __init__(self, timeout=False):
self.oauth_token = oauth.Token(key=OAUTH_KEYS['access_token_key'], secret=OAUTH_KEYS['access_token_secret'])
self.oauth_consumer = oauth.Consumer(key=OAUTH_KEYS['consumer_key'], secret=OAUTH_KEYS['consumer_secret'])
self.conn = None
self.buffer = ''
self.timeout = timeout
self.setup_connection()
def setup_connection(self):
""" Create persistant HTTP connection to Streaming API endpoint using cURL.
"""
if self.conn:
self.conn.close()
self.buffer = ''
self.conn = pycurl.Curl()
# Restart connection if less than 1 byte/s is received during "timeout" seconds
if isinstance(self.timeout, int):
self.conn.setopt(pycurl.LOW_SPEED_LIMIT, 1)
self.conn.setopt(pycurl.LOW_SPEED_TIME, self.timeout)
self.conn.setopt(pycurl.URL, API_ENDPOINT_URL)
self.conn.setopt(pycurl.USERAGENT, USER_AGENT)
# Using gzip is optional but saves us bandwidth.
self.conn.setopt(pycurl.ENCODING, 'deflate, gzip')
self.conn.setopt(pycurl.POST, 1)
self.conn.setopt(pycurl.POSTFIELDS, urllib.urlencode(POST_PARAMS))
self.conn.setopt(pycurl.HTTPHEADER, ['Host: stream.twitter.com',
'Authorization: %s' % self.get_oauth_header()])
# self.handle_tweet is the method that are called when new tweets arrive
self.conn.setopt(pycurl.WRITEFUNCTION, self.handle_tweet)
def get_oauth_header(self):
""" Create and return OAuth header.
"""
params = {'oauth_version': '1.0',
'oauth_nonce': oauth.generate_nonce(),
'oauth_timestamp': int(time.time())}
req = oauth.Request(method='POST', parameters=params, url='%s?%s' % (API_ENDPOINT_URL,
urllib.urlencode(POST_PARAMS)))
req.sign_request(oauth.SignatureMethod_HMAC_SHA1(), self.oauth_consumer, self.oauth_token)
return req.to_header()['Authorization'].encode('utf-8')
def start(self):
""" Start listening to Streaming endpoint.
Handle exceptions according to Twitter's recommendations.
"""
backoff_network_error = 0.25
backoff_http_error = 5
backoff_rate_limit = 60
while True:
self.setup_connection()
try:
self.conn.perform()
except:
# Network error, use linear back off up to 16 seconds
print 'Network error: %s' % self.conn.errstr()
print 'Waiting %s seconds before trying again' % backoff_network_error
time.sleep(backoff_network_error)
backoff_network_error = min(backoff_network_error + 1, 16)
continue
# HTTP Error
sc = self.conn.getinfo(pycurl.HTTP_CODE)
if sc == 420:
# Rate limit, use exponential back off starting with 1 minute and double each attempt
print 'Rate limit, waiting %s seconds' % backoff_rate_limit
time.sleep(backoff_rate_limit)
backoff_rate_limit *= 2
else:
# HTTP error, use exponential back off up to 320 seconds
print 'HTTP error %s, %s' % (sc, self.conn.errstr())
print 'Waiting %s seconds' % backoff_http_error
time.sleep(backoff_http_error)
backoff_http_error = min(backoff_http_error * 2, 320)
def handle_tweet(self, data):
""" This method is called when data is received through Streaming endpoint.
"""
self.buffer += data
if data.endswith('\r\n') and self.buffer.strip():
# complete message received
message = json.loads(self.buffer)
self.buffer = ''
msg = ''
if message.get('limit'):
print 'Rate limiting caused us to miss %s tweets' % (message['limit'].get('track'))
elif message.get('disconnect'):
raise Exception('Got disconnect: %s' % message['disconnect'].get('reason'))
elif message.get('warning'):
print 'Got warning: %s' % message['warning'].get('message')
else:
print 'Got tweet with text: %s' % message.get('text')
if __name__ == '__main__':
ts = TwitterStream()
ts.setup_connection()
ts.start()
please help me to resolve the issue with code