I have a custom module written called sqs.py. The script will do the following:
Get a message from AWS SQS
Get the AWS S3 path to delete
Delete the path
Send a confirmation email to the user
I'm trying to write unit tests for this module that will verify the code will execute as expected and that it will raise exceptions when they do occur.
This means I will need to mock the response from Boto3 calls that I make. My problem is that the code will first establish the SQS client to obtain the message and then a second call to establish the S3 client. I'm not sure how to mock these 2 independent calls and be able to fake a response so I can test my script's functionality. Perhaps my approach is incorrect. At any case, any advice on how to do this properly is appreciated.
Here's how the code looks like:
import boto3
import json
import os
import pprint
import time
import asyncio
import logging
from send_email import send_email
queue_url = 'https://xxxx.queue.amazonaws.com/1234567890/queue'
def shutdown(message):
""" Sends shutdown command to OS """
os.system(f'shutdown +5 "{message}"')
def send_failure_email(email_config: dict, error_message: str):
""" Sends email notification to user with error message attached. """
recipient_name = email_config['recipient_name']
email_config['subject'] = 'Subject: Restore Failed'
email_config['message'] = f'Hello {recipient_name},\n\n' \
+ 'We regret that an error has occurred during the restore process. ' \
+ 'Please try again in a few minutes.\n\n' \
+ f'Error: {error_message}.\n\n' \
try:
send_email(email_config)
except RuntimeError as error_message:
logging.error(f'ERROR: cannot send email to user. {error_message}')
async def restore_s3_objects(s3_client: object, p_bucket_name: str, p_prefix: str):
"""Attempts to restore objects specified by p_bucket_name and p_prefix.
Returns True if restore took place, false otherwise.
"""
is_truncated = True
key_marker = None
key = ''
number_of_items_restored = 0
has_restore_occured = False
logging.info(f'performing restore for {p_bucket_name}/{p_prefix}')
try:
while is_truncated == True:
if not key_marker:
version_list = s3_client.list_object_versions(
Bucket = p_bucket_name,
Prefix = p_prefix)
else:
version_list = s3_client.list_object_versions(
Bucket = p_bucket_name,
Prefix = p_prefix,
KeyMarker = key_marker)
if 'DeleteMarkers' in version_list:
logging.info('found delete markers')
delete_markers = version_list['DeleteMarkers']
for d in delete_markers:
if d['IsLatest'] == True:
key = d['Key']
version_id = d['VersionId']
s3_client.delete_object(
Bucket = p_bucket_name,
Key = key,
VersionId = version_id
)
number_of_items_restored = number_of_items_restored + 1
is_truncated = version_list['IsTruncated']
logging.info(f'is_truncated: {is_truncated}')
if 'NextKeyMarker' in version_list:
key_marker = version_list['NextKeyMarker']
if number_of_items_restored > 0:
has_restore_occured = True
return has_restore_occured
except Exception as error_message:
raise RuntimeError(error_message)
async def main():
if 'AWS_ACCESS_KEY_ID' in os.environ \
and 'AWS_SECRET_ACCESS_KEY' in os.environ \
and os.environ['AWS_ACCESS_KEY_ID'] != '' \
and os.environ['AWS_SECRET_ACCESS_KEY'] != '':
sqs_client = boto3.client(
'sqs',
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
verify=False
)
s3_client = boto3.client(
's3',
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
verify=False
)
else:
sqs_client = boto3.client(
'sqs',
verify=False,
)
s3_client = boto3.client(
's3',
verify=False,
)
received_message = sqs_client.receive_message(
QueueUrl=queue_url,
AttributeNames=['All'],
VisibilityTimeout=10,
WaitTimeSeconds=20, # Wait up to 20 seconds for a message to arrive
)
if 'Messages' in received_message \
and len(received_message['Messages']) > 0:
# NOTE: Initialize email configuration
receipient_email = 'support#example.com'
username = receipient_email.split('#')[0]
fullname_length = len(username.split('.'))
fullname = f"{username.split('.')[0]}" # Group name / First name only
if (fullname_length == 2): # First name and last name available
fullname = f"{username.split('.')[0]} {username.split('.')[1]}"
fullname = fullname.title()
email_config = {
'destination': receipient_email,
'recipient_name': fullname,
'subject': 'Subject: Restore Complete',
'message': ''
}
try:
receipt_handle = received_message['Messages'][0]['ReceiptHandle']
except Exception as error_message:
logging.error(error_message)
send_failure_email(email_config, error_message)
shutdown(f'{error_message}')
try:
data = received_message['Messages'][0]['Body']
data = json.loads(data)
logging.info('A SQS message for a restore has been received.')
except Exception as error_message:
message = f'Unable to obtain and parse message body. {error_message}'
logging.error(message)
send_failure_email(email_config, message)
shutdown(f'{error_message}')
try:
bucket = data['bucket']
prefix = data['prefix']
except Exception as error_message:
message = f'Retrieving bucket name and prefix failed. {error_message}'
logging.error(message)
send_failure_email(email_config, message)
shutdown(f'{error_message}')
try:
logging.info(f'Initiating restore for path: {bucket}/{prefix}')
restore_was_performed = await asyncio.create_task(restore_s3_objects(s3_client, bucket, prefix))
if restore_was_performed is True:
email_config['message'] = f'Hello {fullname},\n\n' \
+ f'The files in the path \'{bucket}/{prefix}\' have been restored. ' \
send_email(email_config)
logging.info('Restore complete. Shutting down.')
else:
logging.info('Path does not require restore. Shutting down.')
shutdown(f'shutdown +5 "Restore successful! System will shutdown in 5 mins"')
except Exception as error_message:
message = f'File restoration failed. {error_message}'
logging.error(message)
send_failure_email(email_config, message)
shutdown(f'{error_message}')
try:
sqs_client.delete_message(
QueueUrl=queue_url,
ReceiptHandle=receipt_handle,
)
except Exception as error_message:
message = f'Deleting restore session from SQS failed. {error_message}'
logging.error(message)
send_failure_email(email_config, message)
shutdown(f'{error_message}')
if __name__ == '__main__':
logging.basicConfig(filename='restore.log',level=logging.INFO)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
The only way I was able to mock Boto3 is rebuilding a small class that represents the actual method structure. This is because Boto3 uses dynamic methods and all the resource level methods are created at runtime.
This might not be industry standard but I wasn't able to get any of the methods I found on the internet to work most of the time and this worked pretty well for me and requires minimal effort (comparing to some of the solutions I found).
class MockClient:
def __init__(self, region_name, aws_access_key_id, aws_secret_access_key):
self.region_name = region_name
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
self.MockS3 = MockS3()
def client(self, service_name, **kwargs):
return self.MockS3
class MockS3:
def __init__(self):
self.response = None # Test your mock data from S3 here
def list_object_versions(self, **kwargs):
return self.response
class S3TestCase(unittest.TestCase):
def test_restore_s3_objects(self):
# Given
bucket = "testBucket" # Test this to something that somewahat realistic
prefix = "some/prefix" # Test this to something that somewahat realistic
env_vars = mock.patch.dict(os.environ, {"AWS_ACCESS_KEY_ID": "abc",
"AWS_SECRET_ACCESS_KEY": "def"})
env_vars.start()
# initialising the Session can be tricy since it has to be imported from
# the module/file that creates the session on actual code rather than
# where's a Session code is. In this case you might have to import from
# main rather than boto3.
boto3.session.Session = mock.Mock(side_effect=[
MockClient(region_name='eu-west-1',
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'])])
s3_client = boto3.client('s3', verify=False)
# When
has_restore_occured = restore_s3_objects(s3_client, bucket, prefix)
# Then
self.assertEqual(has_restore_occured, False) # your expected result set
env_vars.stop()
Related
I'm trying to validate my Slack bot using the example code provided in the Slack documentation here. However, it is returning two different signatures.
def lambda_handler(event, context):
# get slack secret from secrets manager
secret = get_secret()
# needed for creating hmac
qstring = base64.b64decode(event['body']).decode('utf-8')
return validate_request(secret, qstring, event['headers'])
# get slack secret for verification from secrets manager
def get_secret():
secret_name = "<omitted>"
region_name = "<omitted>"
session = boto3.session.Session()
client = session.client(
service_name='secretsmanager',
region_name=region_name
)
try:
get_secret_value_response = client.get_secret_value(
SecretId=secret_name
)
except ClientError as e:
if e.response['Error']['Code'] == 'DecryptionFailureException':
raise e
elif e.response['Error']['Code'] == 'InternalServiceErrorException':
raise e
elif e.response['Error']['Code'] == 'InvalidParameterException':
raise e
elif e.response['Error']['Code'] == 'InvalidRequestException':
raise e
elif e.response['Error']['Code'] == 'ResourceNotFoundException':
raise e
else:
if 'SecretString' in get_secret_value_response:
return get_secret_value_response['SecretString']
else:
return base64.b64decode(get_secret_value_response['SecretBinary'])
# validate bot request
def validate_request(secret, body, headers):
timestamp = headers['x-slack-request-timestamp']
sig_basestring = 'v0:' + timestamp + ':' + body
my_signature = 'v0=' + hmac.new(secret.encode('utf_8'), sig_basestring.encode('utf_8'), hashlib.sha256).hexdigest()
slack_signature = headers['x-slack-signature']
if hmac.compare_digest(my_signature, slack_signature):
return True
else:
return False
Result:
v0=24dde133843073b58084970afe027e3a4dabc1b8d9efc5248a97ad64c6529cee
v0=bf51d6fb9eb56d5c6ea19e866b798903fb0cee67264cb467ee7924bb13571770
Any ideas why? I've verified that the correct token is returned by get_secret() and that the qstring variable contains the correct query parameters as shown in Slack's documentation. The headers all contain the correct values too.
For me the signatures were different because I was formatting the body received from slack request to json before passing to the verify function.
Passing the body as it is resolved my issue.
Format of body to be passed to verify function-
'body': 'token=XXXXXXXXXX&team_id=XXXXXXX&team_domain=XXXXXX&channel_id=XXXXX&channel_name=XXXXXX&user_id=XXX&user_name=XXXX&command=XXXXX&text=XXXX&api_app_id=XXXXXX&is_enterprise_install=false&response_url=XXXXX&trigger_id=XXXXXXXX'
I am working on a use case to creating groups in AD through PyAD and create folder and groups for that folder through flask.
I am using for loop for passing arguments and returning responses. If the group exists code should not create if else it should create and then move on to create folder and set permissions.
But the logic works fine for for the first group passed in request, but 2nd one is not getting into the loop.
Facing issues making it work through flask and handle responses. Is there is a way to achive it, please help.
app = Flask(__name__)
api = Api(app)
#Class to create fileshare
class Test(Resource):
def post(self):
pythoncom.CoInitialize()
# Get JSON arguments from Payload shared NAS path, directorname groupname with read access and right access
parentdir = request.json.get("shareUNCPath")
dirname = request.json.get("shareFolderName")
readGroup = request.json.get("readGroup")
writeGroup = request.json.get("writeGroup")
domainName = request.json.get("domain")
groupList = [readGroup,writeGroup]
#for gn in groupList:
try:
j=(len(groupList))+1
if readGroup == writeGroup:
j=(len(groupList))-1
#for gn in len(groupList):
for i in range(4):
groupName = groupList[i]
pyad.set_defaults(username="username", password="password", ldap_server="ldapServer")
rGroup = adgroup.ADGroup.from_cn(groupName)
logging.debug("read group {} available in AD ".format(groupName))
if __name__ == "__main__":
os.makedirs(path)
igroup, domain, type = win32security.LookupAccountName (domainName, groupName)
sd = win32security.GetFileSecurity(path, win32security.DACL_SECURITY_INFORMATION)
dacl = sd.GetSecurityDescriptorDacl()
logging.debug("Domain1 {}, Group1 {}".format(domainName, groupName))
if groupName in readGroup:
dacl.AddAccessAllowedAce(win32security.ACL_REVISION,con.GENERIC_READ, igroup)
if groupName in writeGroup:
dacl.AddAccessAllowedAce(win32security.ACL_REVISION,con.GENERIC_WRITE, igroup)
isdir = os.path.isdir(path)
if isdir == True:
sd.SetSecurityDescriptorDacl(1, dacl, 0)
win32security.SetFileSecurity(path, win32security.DACL_SECURITY_INFORMATION, sd)
dacl = sd.GetSecurityDescriptorDacl()
cnt=dacl.GetAceCount()
for i in range(0, cnt):
rev, access, usersid = dacl.GetAce(i)
user, group, type = win32security.LookupAccountSid(domainName, usersid)
details = ('Group: {}/{}'.format(group, user), rev, access)))
resp = Response('Successfully created file share {}. Details {}'.format(dirname, details))
print (resp)
resp.status_code = 200
return resp
except Exception as e:
errormsg = str(e)
print (errormsg)
if "The server is not operational" in errormsg:
resp = Response('AD operation failed, unable to connect to Active Directory. Error - {}'.format(e))
print (resp)
resp.status_code = 301
return resp
else:
try:
for i in range(4):
groupName = groupList[i]
pyad.set_defaults(username="username", password="pasword",ldap_server="ldapServer")
ou = pyad.adcontainer.ADContainer.from_dn(group_OU)
rGroup = adgroup.ADGroup.create(
name=groupName,
security_enabled = True,
scope=groupScope,
container_object=ou,
optional_attributes={"description": description}
)
if rGroup.Displayname == (groupName):
if __name__ == "__main__":
os.makedirs(path)
#groupr = win32security.LookupAccountName ("", readGroup)
a.logon()
time.sleep(5)
igroup, domain, type = win32security.LookupAccountName (domainName, groupName)
sd = win32security.GetFileSecurity(path, win32security.DACL_SECURITY_INFORMATION)
#dacl = win32security.ACL()
dacl = sd.GetSecurityDescriptorDacl()
#acl = pywintypes.ACL()
#set permessions for readGroup with GENERIC_READ level permessions
#dacl.AddAccessAllowedAce(win32security.ACL_REVISION,con.GENERIC_READ, groupr)
if groupName in readGroup:
dacl.AddAccessAllowedAceEx(win32security.ACL_REVISION,con.OBJECT_INHERIT_ACE|con.CONTAINER_INHERIT_ACE,con.GENERIC_READ|con.GENERIC_EXECUTE, igroup)
if groupName in writeGroup:
dacl.AddAccessAllowedAce(win32security.ACL_REVISION,con.GENERIC_WRITE, igroup)
isdir = os.path.isdir(path)
if isdir == True:
sd.SetSecurityDescriptorDacl(1, dacl, 0)
win32security.SetFileSecurity(path, win32security.DACL_SECURITY_INFORMATION, sd)
dacl = sd.GetSecurityDescriptorDacl()
cnt=dacl.GetAceCount()
for i in range(0, cnt):
rev, access, usersid = dacl.GetAce(i)
user, group, type = win32security.LookupAccountSid(domainName, usersid)
details = ('Group: {}/{}'.format(group, user), rev, access)
#return ("Success Fileshare created: {} ".format(dirname))
resp = Response('Successfully created file share {}. Details {}'.format(dirname, details))
print (resp)
resp.status_code = 200
return resp
except Exception as e:
print(e)
resp = Response('AD operation failed, unable to create to group {}. Error - {}'.format(groupName, e))
print (resp)
resp.status_code = 302
return resp
api.add_resource(Test, '/test')
if __name__ == "__main__":
#context = ('local.crt', 'local.key')#certificate and key files
app.run(port="7050", host="0.0.0.0", use_reloader=True)
I reviewed your code. There are two things that should changed.
You use i as the loop variable for the outer and inner loop
In the first loop, you use an exception to trigger the group creation. This exits the loop and no more groups are processed. You should move the exception block inside the range(4) loop.
Here is your code with comments.
class Test(Resource):
def post(self):
.......
try:
..........
for i in range(4): # using i as loop variable, loop will exit if exception
........
if __ name __ == "__ main __": # if group exists, update permissions, throws exception if group does not exist
........
if isdir == True:
........
for i in range(0, cnt): # using i as loop variable, again
.........
# here is the problem - if the first group does not exist, an exception is thrown and the other groups are not processed
except Exception as e: # group does not exist, must add # You should move this inside the for loop
............
try:
for i in range(4): # using i as loop variable
...........
if rGroup.Displayname == (groupName):
if __ name __ == "__main__":
.........
if isdir == True:
........
for i in range(0, cnt): # using i as loop variable, again
..........
To clarify, the overall logic should be like this:
for i in range(4): # each group
try:
# update permissions
except Exception as e:
# add new group
As a side note, try to check if the group exists without using the try\except block. Exceptions should not be used in normal program flow.
I am using below code with flask and name is app1.py
app = Flask(__name__)
api = Api(app)
#Class to create fileshare
class FileShare(Resource):
def post(self):
# Get JSON arguments from Payload shared NAS path, directorname groupname with read access and right access
parentdir = request.json.get("shareUNCPath")
dirname = request.json.get("shareFolderName")
readGroup = request.json.get("readGroup")
writeGroup = request.json.get("writeGroup")
#Create shared path with UNC Path and sharefolder name
path = os.path.join(parentdir, dirname)
# Access the NAS path through NAS credentails
class Impersonate:
def __init__(self,user,password):
#Update domain to access the shared NAS
self.domain_name = "<domain>"
self.user = user
self.password = password
logging.debug("Credentials Received: {} ".format(self.user))
def logon(self):
self.handle=win32security.LogonUser(self.user,self.domain_name,self.password,win32con.LOGON32_LOGON_INTERACTIVE,win32con.LOGON32_PROVIDER_DEFAULT)
win32security.ImpersonateLoggedOnUser(self.handle)
def logoff(self):
win32security.RevertToSelf() #terminates impersonation
self.handle.Close() #guarantees cleanup
if __name__ == "__main__":
#update username and password of the NAS path below within quotes
a=Impersonate('username','password')
try:
a.logon() #Logon to NAS path with supplied credentails.
try:
logging.debug("Sucessfully connectd to NAS path {} ".format(parentdir))
# makedirs create directory recursively
os.makedirs(path)
#Set Permissions for the share folder for respective group
#Get the value of the groups stored in groupr and groupw as variables
try:
groupr, domain, type = win32security.LookupAccountName ("", readGroup)
sd = win32security.GetFileSecurity(path, win32security.DACL_SECURITY_INFORMATION)
dacl = sd.GetSecurityDescriptorDacl()
#set permessions for readGroup with GENERIC_READ level permessions
dacl.AddAccessAllowedAce(win32security.ACL_REVISION,win32con.GENERIC_READ, groupr)
except win32security.error as e:
resp = Response('Sucessfully created fileshare {} however setting group permession for the group {} failed. Error {}'.format(dirname, readGroup, e))
print (resp)
resp.status_code = 301
return resp
sd.SetSecurityDescriptorDacl(1, dacl, 0)
win32security.SetFileSecurity(path, win32security.DACL_SECURITY_INFORMATION, sd)
try:
groupw, domain, type = win32security.LookupAccountName ("", writeGroup)
sd = win32security.GetFileSecurity(path, win32security.DACL_SECURITY_INFORMATION)
dacl = sd.GetSecurityDescriptorDacl()
#set permessions for writeGroup with GENERIC_WRITE level permessions
dacl.AddAccessAllowedAce(win32security.ACL_REVISION,win32con.GENERIC_WRITE, groupw)
except win32security.error as e:
resp = Response('Sucessfully created fileshare {} however setting group permession for the group {} failed. Error {}'.format(dirname, writeGroup, e))
print (resp)
resp.status_code = 301
return resp
sd.SetSecurityDescriptorDacl(1, dacl, 0)
win32security.SetFileSecurity(path, win32security.DACL_SECURITY_INFORMATION, sd)
dacl = sd.GetSecurityDescriptorDacl()
cnt=dacl.GetAceCount()
for i in range(0, cnt):
rev, access, usersid = dacl.GetAce(i)
user, group, type = win32security.LookupAccountSid('', usersid)
details = ('Group: {}/{}'.format(group, user), rev, access)
#return ("Success Fileshare created: {} ".format(dirname))
resp = Response('Successfully created file share {}. Details {}'.format(dirname, details))
print (resp)
resp.status_code = 200
return resp
except OSError as error:
print(error)
resp = Response('NAS operatoon failed, {} fileshare creation failed. Error - {}'.format(dirname, error))
print (resp)
resp.status_code = 300
return resp
#return ("Fileshare creation failed: {} ".format(dirname))
except Exception as error1:
print(error1)
logging.error("Failed to connect to NASa path{}, Error: {} ".format(parentdir, error1))
resp = Response('NAS Operation failed, could not connect to UNC Shared path. Error{}'.format(error1))
print (resp)
resp.status_code = 201
return resp
a.logoff()
api.add_resource(Create, '/test')
if __name__ == "__main__":
app.run(port=5001, host="0.0.0.0", use_reloader=True)
I have created waitress_server.py with below app1.py is called in below code.
Just executing app1.py works fine, however when tried with waitress i see HTTP post through postman is received as NULL.
I am trying to use waitress for my production setup.
from waitress import serve
import app1
serve(app1.app, host='0.0.0.0', port=5001)
When i make a postman call i see response as null. Please help me if I am doing right.
You have forgotten to return a Response in case of no given error.
Specifically this part:
if __name__ == "__main__":
dacl.AddAccessAllowedAce(win32security. ACL_REVISION,win32con.GENERIC_READ, groupr) #os.makedirs(path)
Is missing a return statement. The postman call returns None because functions return None by default.
You need to add a Response after the above mentioned code and also return it after all fallbacks.
PS: thank your for adding the code
Gmail decided that SMTP was too simple so they had to block it and setup their own API with all the weird requirements around it. This script which I was trying to use has now become outdated and broken. In an attempt to use it anyway I tried to rewrite it:
"""
Checks stock on specified items at Microcenter store locations,
and sends email notifications when changes are detected.
Applicably, it helps the user obtain rare items during shortages.
"""
from aiohttp import ClientSession
from async_timeout import timeout
from getpass import getpass
from re import search
from smtplib import SMTP
import asyncio
import base64
from email.mime.audio import MIMEAudio
from email.mime.base import MIMEBase
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import mimetypes
import os
import pickle
import os.path
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
class Item:
"""
Class for containing state of individual items; methods update state
by awaiting update().
Item does not need to be directly instantiated; Store will create one
per provided url.
"""
def __init__(self, storeNum, url):
self.storeNum, self.url = storeNum, url
self.sku = self.price = self.stock = None
self.stockChanged = self.priceChanged = False
self.loop = asyncio.get_event_loop()
def __str__(self):
stock = 'in' if self.stock else 'out of'
return f'SKU {self.sku} is {stock} stock for {self.price} at Microcenter {self.storeNum}\n{self.url}\n'
async def pull(self):
async with ClientSession() as session:
async with timeout(10):
async with session.get(self.url, params={'storeSelected': self.storeNum}) as response:
return await response.text()
#staticmethod
def parse_lines(page):
for var in ['SKU', 'inStock', 'productPrice']:
reply = search(f"(?<='{var}':').*?(?=',)", page)
if reply:
yield reply.group()
#staticmethod
def compare(new, old):
return (new != old and old is not None)
async def update(self):
data = tuple(self.parse_lines(await self.pull()))
if not data or any(data) is None:
raise ValueError('Data missing from request or store number invalid')
self.sku, stock, price = int(data[0]), data[1] is 'True', float(data[2])
self.stockChanged, self.priceChanged = self.compare(stock, self.stock), self.compare(price, self.price)
self.stock, self.price = stock, price
class Store:
"""
Periodically checks a given list of urls for stock changes
A store number is required to get accurate stock numbers.
The default store number is set to the North Dallas/Richardson, TX location.
Also required is valid email account information for notifications.
If a recipient address is not provided, the user will be prompted for one.
If the prompt is empty, notifications are sent from the sender
address to itself. Providing an empty string for recipient is a valid
argument to enable loopback operation, as only a value of None
will trigger a prompt.
The default time between checks is 15 minutes. This value should
be at least a few minutes, to avoid being blacklisted by the
server, though this class enforces no such limit. To change the
time period, provide a value in minutes to self.run(minutes).
Setting debug to True enables false positives for testing
"""
def __init__(
self, storeNum=131, sender=None,
recipient=None, debug=True, service=None
):
self.storeNum = storeNum
self.items, self.newInStock, self.totalInStock = set(), 0, 0
self.debug = debug
if not sender:
self.sender = input('Enter sender email address: ').lstrip().rstrip()
else:
self.sender = sender
if recipient is None:
prompted = input('Enter recipient email address (leave blank for loopback): ').lstrip().rstrip()
if not prompted:
self.recipient = self.sender
else:
self.recipient = prompted
else:
self.recipient = self.sender
#Google API BULLSHIT
SCOPES = ['https://www.googleapis.com/auth/gmail.compose','https://www.googleapis.com/auth/gmail.readonly']
creds = None
# The file token.pickle stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
if os.path.exists('token.pickle'):
with open('token.pickle', 'rb') as token:
creds = pickle.load(token)
# If there are no (valid) credentials available, let the user log in.
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(
'credentials.json', SCOPES)
creds = flow.run_local_server(port=0)
# Save the credentials for the next run
with open('token.pickle', 'wb') as token:
pickle.dump(creds, token)
self.service = build('gmail', 'v1', credentials=creds)
# Call the Gmail API
results = self.service.users().labels().list(userId='me').execute()
labels = results.get('labels', [])
if not labels:
print('No labels found.')
else:
print('Labels:')
for label in labels:
print((label['name']))
self.loop = asyncio.get_event_loop()
def __str__(self):
return '\n'.join(item.__str__() for item in self.items)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.loop.close()
#property
def storeNum(self):
return self._storeNum
#storeNum.setter
def storeNum(self, val):
"""
Check to see if value is formatted properly
storeNum must be sent as a string, but should contain an integer.
"""
assert isinstance(val, (int, str)), 'Store number must be an integer or string of integer'
try:
num = int(val)
except:
raise
else:
self._storeNum = str(num)
#property
def sender(self):
return self._sender
#sender.setter
def sender(self, val):
assert val is not None, 'Sender address cannot be empty'
assert isinstance(val, str), 'Must be str'
self._sender = val
def run(self, minutes=5):
run = asyncio.ensure_future(self.check(minutes))
self.loop.run_forever()
async def check(self, minutes=5):
assert isinstance(minutes, (int, float)), 'Minutes must be an integer or float'
seconds = minutes * 60
while True:
print('Checking stock...')
await self.update()
if self.newInStock:
print('New items available')
msg = email_message()
print("message created")
self.send_email(msg)
print("email send attempted")
#if sent:
#print('Recipient notified of stock changes')
else:
print('Stock unchanged')
await asyncio.sleep(seconds)
def add_interactive(self):
entry = True
while entry:
entry = eval(input('Add one or more URLs separated by spaces, or leave blank to complete: '))
try:
urls = entry.split()
except:
if entry and 'http' in entry:
self.add(entry.lstrip().rstrip())
else:
self.add(*urls)
def add(self, *urls):
for url in urls:
assert isinstance(url, str), 'URL must be a string'
if url not in (item.url for item in self.items):
new = Item(self.storeNum, url)
self.loop.run_until_complete(new.update())
self.items.add(new)
def remove(self, *urls):
for url in urls:
assert isinstance(url, str), 'URL must be a string'
self.items = set([item for item in self.items if item.url not in urls])
def email_message(self):
if self.debug:
new = self.items
else:
new = tuple([item for item in self.items if item.stockChanged])
message_text = '\n'.join(item.__str__() for item in new)
print(message_text)
#Create message container
message = MIMEMultipart('alternative') # needed for both plain & HTML (the MIME type is multipart/alternative)
message['Subject'] = self.email_subject()
print("set Subject")
message['From'] = self.sender
print("set sender")
message['To'] = self.recipient
print("set recipient")
#Create the body of the message (a plain-text and an HTML version)
message.attach(MIMEText(message_text, 'plain'))
print("attached plaintext")
message.attach(MIMEText(message_text, 'html'))
print("attached html")
raw_message_no_attachment = base64.urlsafe_b64encode(message.as_bytes())
print("encoded b64")
raw_message_no_attachment = raw_message_no_attachment.decode()
print("decoded raw")
body = {'raw': raw_message_no_attachment}
print("set body")
return body
def email_subject(self):
return f'({self.newInStock} new, {self.totalInStock} total) items in stock at Microcenter {self.storeNum}'
def send_email(self, msgOBJ):
message = msgOBJ
print("message encoded")
try:
message_sent = (self.service.users().messages().send(userId='me', body=message).execute())
message_id = message_sent['id']
# print(attached_file)
print (f'Message sent (without attachment) \n\n Message Id: {message_id}\n\n Message:\n\n {message_text_plain}')
# return body
return True
except errors.HttpError as error:
print (f'An error occurred: {error}')
return False
async def update(self):
for item in self.items:
await item.update()
if self.debug:
self.newInStock = self.totalInStock = len(self.items)
else:
self.newInStock = sum(item.stockChanged for item in self.items)
self.totalInStock = sum(item.stock for item in self.items)
class Clerk(Store):
"""
Further abstraction and automation of Store
Instantiate Clerk with a list of urls as arguments
and an optional store number as a keyword argument.
Clerk exists to be able to start and run a Store in one line.
The user will be prompted for email account information.
"""
def __init__(self, *urls, storeNum=131):
super().__init__(storeNum=storeNum)
if urls:
super().add(*urls)
else:
super().add_interactive()
super().run()
Clerk("https://www.microcenter.com/product/616858/amd-ryzen-9-3950x-35ghz-16-core-am4-boxed-processor", storeNum=155)
I wrote this in a way that is Python 3.6 compatible and Gmail API friendly so it'll actually work. However, upon calling the Store.email_message method (which is supposed to create and return the necessary b64 encoded message object) nothing happens, not one of the prints spaced throughout it is called and no error is returned either. It just stops there.
I initially tried the code from the examples in the Gmail API Documentation, but that didn't work, so then i went searching through the web until I decided to stop with the code I got here (code stolen from their send_Message_without_attachment and create_message_without_attachment functions) and ask for help.
Edit
I followed the advice of the answer I got and changed the email_message function to
def email_message(self):
if self.debug:
new = self.items
else:
new = tuple([item for item in self.items if item.stockChanged])
message_text = '\n'.join(item.__str__() for item in new)
print(message_text)
#Create message container
message = MIMEMultipart('alternative') # needed for both plain & HTML (the MIME type is multipart/alternative)
message['Subject'] = self.email_subject()
message['From'] = self.sender
message['To'] = self.recipient
#Create the body of the message (a plain-text and an HTML version)
message.attach(MIMEText(message_text, 'plain'))
message.attach(MIMEText(message_text, 'html'))
raw_message_no_attachment = urlsafe_b64encode(bytes(message))
raw_message_no_attachment = raw_message_no_attachment.decode()
body = {'raw': raw_message_no_attachment}
return body
That said it still gives no error and doesn't even get to print the message text when it gets to the point where it's called, so I'm still pretty lost.
For the encoding you have to change your import and use like this:
Import:
from base64 import urlsafe_b64encode
Use:
encode = urlsafe_b64encode(bytes(message))
For the scopes using this one is more than enough:
SCOPES = ['https://mail.google.com/']
Remember to delete and renew the token.pickle every time you change the scopes.
Be sure that the API credentials are Ok.
Background:
I would like to integrate yowsup to my home automation project. I have seen a simple sample on how to receive messages and after some minor changes it is working fine.
Issue:
My problem starts when it comes to integrate the send message feature. Those are the two files I am using:
run.py
from layer import EchoLayer
from yowsup.layers.auth import YowAuthenticationProtocolLayer
from yowsup.layers.protocol_messages import YowMessagesProtocolLayer
from yowsup.layers.protocol_receipts import YowReceiptProtocolLayer
from yowsup.layers.protocol_acks import YowAckProtocolLayer
from yowsup.layers.protocol_presence import YowPresenceProtocolLayer
from yowsup.layers.network import YowNetworkLayer
from yowsup.layers.coder import YowCoderLayer
from yowsup.common import YowConstants
from yowsup.layers import YowLayerEvent
from yowsup.stacks import YowStack, YOWSUP_CORE_LAYERS
from yowsup import env
CREDENTIALS = ("phone", "pwd")
if __name__ == "__main__":
layers = (
EchoLayer,
(YowAuthenticationProtocolLayer, YowMessagesProtocolLayer, YowReceiptProtocolLayer, YowAckProtocolLayer, YowPresenceProtocolLayer)
) + YOWSUP_CORE_LAYERS
stack = YowStack(layers)
# Setting credentials
stack.setProp(YowAuthenticationProtocolLayer.PROP_CREDENTIALS, CREDENTIALS)
# WhatsApp server address
stack.setProp(YowNetworkLayer.PROP_ENDPOINT, YowConstants.ENDPOINTS[0])
stack.setProp(YowCoderLayer.PROP_DOMAIN, YowConstants.DOMAIN)
stack.setProp(YowCoderLayer.PROP_RESOURCE, env.CURRENT_ENV.getResource())
# Sending connecting signal
stack.broadcastEvent(YowLayerEvent(YowNetworkLayer.EVENT_STATE_CONNECT))
# Program main loop
stack.loop()
layer.py
from yowsup.layers.interface import YowInterfaceLayer, ProtocolEntityCallback
from yowsup.layers.protocol_messages.protocolentities import TextMessageProtocolEntity
from yowsup.layers.protocol_receipts.protocolentities import OutgoingReceiptProtocolEntity
from yowsup.layers.protocol_acks.protocolentities import OutgoingAckProtocolEntity
from yowsup.layers.protocol_presence.protocolentities import PresenceProtocolEntity
import threading
import logging
logger = logging.getLogger(__name__)
class EchoLayer(YowInterfaceLayer):
#ProtocolEntityCallback("message")
def onMessage(self, messageProtocolEntity):
#send receipt otherwise we keep receiving the same message over and over
print str(messageProtocolEntity.getFrom()) + ' - ' + str(messageProtocolEntity.getBody())
receipt = OutgoingReceiptProtocolEntity(messageProtocolEntity.getId(), messageProtocolEntity.getFrom())
self.toLower(receipt)
#ProtocolEntityCallback("send_message")
def sendMessage(self, destination, message, messageProtocolEntity):
outgoingMessageProtocolEntity = TextMessageProtocolEntity(
message,
to = destination + "#s.whatsapp.net")
self.toLower(outgoingMessageProtocolEntity)
#ProtocolEntityCallback("receipt")
def onReceipt(self, entity):
ack = OutgoingAckProtocolEntity(entity.getId(), "receipt", "delivery")
self.toLower(ack)
# List of (jid, message) tuples
PROP_MESSAGES = "org.openwhatsapp.yowsup.prop.sendclient.queue"
def __init__(self):
super(EchoLayer, self).__init__()
self.ackQueue = []
self.lock = threading.Condition()
#ProtocolEntityCallback("success")
def onSuccess(self, successProtocolEntity):
self.lock.acquire()
for target in self.getProp(self.__class__.PROP_MESSAGES, []):
phone, message = target
if '#' in phone:
messageEntity = TextMessageProtocolEntity(message, to = phone)
elif '-' in phone:
messageEntity = TextMessageProtocolEntity(message, to = "%s#g.us" % phone)
else:
messageEntity = TextMessageProtocolEntity(message, to = "%s#s.whatsapp.net" % phone)
self.ackQueue.append(messageEntity.getId())
self.toLower(messageEntity)
self.lock.release()
#ProtocolEntityCallback("ack")
def onAck(self, entity):
self.lock.acquire()
if entity.getId() in self.ackQueue:
self.ackQueue.pop(self.ackQueue.index(entity.getId()))
if not len(self.ackQueue):
logger.info("Message sent")
#raise KeyboardInterrupt()
self.lock.release()
Questions:
Where am I supposed to call the send_message method, so I can send messages wherever I need it?
Is there a regular event (triggering every second or something) which I could use to send my messages?
#ProtocolEntityCallback("send_message")
def sendMessage(self, destination, message, messageProtocolEntity):
outgoingMessageProtocolEntity = TextMessageProtocolEntity(
message,
to = destination + "#s.whatsapp.net")
self.toLower(outgoingMessageProtocolEntity)
In the avove code sendMessage to be called, protocolEntity.getTag() == "send_message" has to be True. You don't need it to send message.
layer.py
from yowsup.layers.interface import YowInterfaceLayer, ProtocolEntityCallback
from yowsup.layers.protocol_messages.protocolentities import TextMessageProtocolEntity
from yowsup.layers.protocol_receipts.protocolentities import OutgoingReceiptProtocolEntity
from yowsup.layers.protocol_acks.protocolentities import OutgoingAckProtocolEntity
from yowsup.layers.protocol_presence.protocolentities import PresenceProtocolEntity
import threading
import logging
logger = logging.getLogger(__name__)
recv_msg = []
class EchoLayer(YowInterfaceLayer):
def __init__(self):
super(EchoLayer, self).__init__()
self.ackQueue = []
self.lock = threading.Condition()
#ProtocolEntityCallback("message")
def onMessage(self, messageProtocolEntity):
if messageProtocolEntity.getType() == 'text':
recv_msg.append((messageProtocolEntity.getFrom(),messageProtocolEntity.getBody()))
#send receipt otherwise we keep receiving the same message over and over
receipt = OutgoingReceiptProtocolEntity(messageProtocolEntity.getId(), messageProtocolEntity.getFrom())
self.toLower(receipt)
#ProtocolEntityCallback("receipt")
def onReceipt(self, entity):
ack = OutgoingAckProtocolEntity(entity.getId(), "receipt", "delivery")
self.toLower(ack)
# List of (jid, message) tuples
PROP_MESSAGES = "org.openwhatsapp.yowsup.prop.sendclient.queue"
#ProtocolEntityCallback("success")
def onSuccess(self, successProtocolEntity):
self.lock.acquire()
for target in self.getProp(self.__class__.PROP_MESSAGES, []):
phone, message = target
if '#' in phone:
messageEntity = TextMessageProtocolEntity(message, to = phone)
elif '-' in phone:
messageEntity = TextMessageProtocolEntity(message, to = "%s#g.us" % phone)
else:
messageEntity = TextMessageProtocolEntity(message, to = "%s#s.whatsapp.net" % phone)
self.ackQueue.append(messageEntity.getId())
self.toLower(messageEntity)
self.lock.release()
#ProtocolEntityCallback("ack")
def onAck(self, entity):
self.lock.acquire()
if entity.getId() in self.ackQueue:
self.ackQueue.pop(self.ackQueue.index(entity.getId()))
if not len(self.ackQueue):
self.lock.release()
logger.info("Message sent")
raise KeyboardInterrupt()
self.lock.release()
To send message define a function send_message in the stack run.py. You can also import run.py and use it's function from other script.
from layer import EchoLayer, recv_msg
CREDENTIALS = ("phone", "pwd")
def send_message(destination, message):
'''
destination is <phone number> without '+'
and with country code of type string,
message is string
e.g send_message('11133434343','hello')
'''
messages = [(destination, message)]
layers = (EchoLayer,
(YowAuthenticationProtocolLayer,
YowMessagesProtocolLayer,
YowReceiptProtocolLayer,
YowAckProtocolLayer,
YowPresenceProtocolLayer)
) + YOWSUP_CORE_LAYERS
stack = YowStack(layers)
stack.setProp(EchoLayer.PROP_MESSAGES, messages)
stack.setProp(YowAuthenticationProtocolLayer.PROP_PASSIVE, True)
# Setting credentials
stack.setProp(YowAuthenticationProtocolLayer.PROP_CREDENTIALS, CREDENTIALS)
# WhatsApp server address
stack.setProp(YowNetworkLayer.PROP_ENDPOINT, YowConstants.ENDPOINTS[0])
stack.setProp(YowCoderLayer.PROP_DOMAIN, YowConstants.DOMAIN)
stack.setProp(YowCoderLayer.PROP_RESOURCE, env.CURRENT_ENV.getResource())
# Sending connecting signal
stack.broadcastEvent(YowLayerEvent(YowNetworkLayer.EVENT_STATE_CONNECT))
try:
# Program main loop
stack.loop()
except AuthError as e:
print('Authentication error %s' % e.message)
sys.exit(1)
def recv_message():
layers = ( EchoLayer,
(YowAuthenticationProtocolLayer, YowMessagesProtocolLayer,
YowReceiptProtocolLayer, YowAckProtocolLayer,
YowPresenceProtocolLayer)
) + YOWSUP_CORE_LAYERS
stack = YowStack(layers)
# Setting credentials
stack.setProp(YowAuthenticationProtocolLayer.PROP_CREDENTIALS, CREDENTIALS)
# WhatsApp server address
stack.setProp(YowNetworkLayer.PROP_ENDPOINT, YowConstants.ENDPOINTS[0])
stack.setProp(YowCoderLayer.PROP_DOMAIN, YowConstants.DOMAIN)
stack.setProp(YowCoderLayer.PROP_RESOURCE, env.CURRENT_ENV.getResource())
# Sending connecting signal
stack.broadcastEvent(YowLayerEvent(YowNetworkLayer.EVENT_STATE_CONNECT))
try:
# Program main loop
stack.loop()
except AuthError as e:
print('Authentication error %s' % e.message)
sys.exit(1)
if __name__ == '__main__':
if len(sys.argv) == 1:
print('%s send number message\nrecv\n' % sys.argv[0])
sys.exit(1)
if sys.argv[1] == 'send':
try:
send_message(sys.argv[2],sys.argv[3])
except KeyboardInterrupt:
print('closing')
sys.exit(0)
if sys.argv[1] == 'recv':
try:
recv_message()
except KeyboardInterrupt:
print('closing')
sys.exit(0)
for m in recv_msg:
print('From %s:\n%s\n' % m)
Now you can send message by calling send_message('1234567890','Howdy') and recieve message by calling recv_message().