I've been updating some scripts to the new Python Gmail API. However, I am confused as how to update the following so that I only retrieve messages from yesterday. Can anyone show me how to do this?
The only way I can currently see is to loop through all messages and only parse those with epochs in the correct time range. However, that seems horribly inefficient if I have 1000's of messages. There must be a more efficient way to do this.
from apiclient import discovery
import oauth2client
from oauth2client import client
from oauth2client import tools
import os
import httplib2
import email
from apiclient.http import BatchHttpRequest
import base64
from bs4 import BeautifulSoup
import re
import datetime
try:
import argparse
flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
flags = None
SCOPES = 'https://www.googleapis.com/auth/gmail.readonly'
CLIENT_SECRET_FILE = '/Users/sokser/Downloads/client_secret.json'
APPLICATION_NAME = 'Gmail API Python Quickstart'
def get_credentials():
"""Gets valid user credentials from storage.
If nothing has been stored, or if the stored credentials are invalid,
the OAuth2 flow is completed to obtain the new credentials.
Returns:
Credentials, the obtained credential.
"""
home_dir = os.path.expanduser('~')
credential_dir = os.path.join(home_dir, '.credentials')
if not os.path.exists(credential_dir):
os.makedirs(credential_dir)
credential_path = os.path.join(credential_dir,
'gmail-python-quickstart.json')
store = oauth2client.file.Storage(credential_path)
credentials = store.get()
if not credentials or credentials.invalid:
flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
flow.user_agent = APPLICATION_NAME
if flags:
credentials = tools.run_flow(flow, store, flags)
else: # Needed only for compatibility with Python 2.6
credentials = tools.run(flow, store)
print('Storing credentials to ' + credential_path)
return credentials
def visible(element):
if element.parent.name in ['style', 'script', '[document]', 'head', 'title']:
return False
elif re.match('<!--.*-->', str(element)):
return False
return True
def main():
"""Shows basic usage of the Gmail API.
Creates a Gmail API service object and outputs a list of label names
of the user's Gmail account.
"""
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
service = discovery.build('gmail', 'v1', http=http)
#Get yesterdays date and the epoch time
yesterday = datetime.date.today() - datetime.timedelta(1)
unix_time= int(yesterday.strftime("%s"))
messages = []
message = service.users().messages().list(userId='me').execute()
for m in message['messages']:
#service.users().messages().get(userId='me',id=m['id'],format='full')
message = service.users().messages().get(userId='me',id=m['id'],format='raw').execute()
epoch = int(message['internalDate'])/1000
msg_str = str(base64.urlsafe_b64decode(message['raw'].encode('ASCII')),'utf-8')
mime_msg = email.message_from_string(msg_str)
#print(message['payload']['parts'][0]['parts'])
#print()
mytext = None
for part in mime_msg.walk():
mime_msg.get_payload()
#print(part)
#print()
if part.get_content_type() == 'text/plain':
soup = BeautifulSoup(part.get_payload(decode=True))
texts = soup.findAll(text=True)
visible_texts = filter(visible,texts)
mytext = ". ".join(visible_texts)
if part.get_content_type() == 'text/html' and not mytext:
mytext = part.get_payload(decode=True)
print(mytext)
print()
if __name__ == '__main__':
main()
You can pass queries to the messages.list method that searches for messages within a date range. You can actually use any query supported by Gmail's advanced search.
You do this, which will just return messages.
message = service.users().messages().list(userId='me').execute()
But can do this to search for messages sent yesterday, by passing the q keyword argument, and a query specifying the before: and after: keywords.
from datetime import date, timedelta
today = date.today()
yesterday = today - timedelta(1)
# do your setup...
user_id = 'user email address'
# Dates have to formatted in YYYY/MM/DD format for gmail
query = "before: {0} after: {1}".format(today.strftime('%Y/%m/%d'),
yesterday.strftime('%Y/%m/%d'))
response = service.users().messages().list(userId=user_id,
q=query).execute()
# Process the response for messages...
You can also try this against their GMail messages.list reference page.
Related
I'm working on python script which captures all my events in google calendar. this code will show the list of all the events in a day.
Below is the code.
from __future__ import print_function
import httplib2
import os
import datetime
from apiclient import discovery
import oauth2client
from oauth2client import client
from oauth2client import tools
from oauth2client.file import Storage
import datetime
try:
import argparse
flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
flags = None
# If modifying these scopes, delete your previously saved credentials
# at ~/.credentials/calendar-python-quickstart.json
SCOPES = 'https://www.googleapis.com/auth/calendar.readonly'
CLIENT_SECRET_FILE = 'credentials.json'
APPLICATION_NAME = 'Google Calendar API Python Quickstart'
def get_credentials():
"""Gets valid user credentials from storage.
If nothing has been stored, or if the stored credentials are invalid,
the OAuth2 flow is completed to obtain the new credentials.
Returns:
Credentials, the obtained credential.
"""
home_dir = os.path.expanduser('~')
credential_dir = os.path.join(home_dir, '.credentials')
if not os.path.exists(credential_dir):
os.makedirs(credential_dir)
credential_path = os.path.join(credential_dir,
'calendar-python-quickstart.json')
store = oauth2client.file.Storage(credential_path)
credentials = store.get()
if not credentials or credentials.invalid:
flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
flow.user_agent = APPLICATION_NAME
if flags:
credentials = tools.run_flow(flow, store, flags)
else: # Needed only for compatibility with Python 2.6
credentials = tools.run(flow, store)
print('Storing credentials to ' + credential_path)
return credentials
def main():
"""Shows basic usage of the Google Calendar API.
Creates a Google Calendar API service object and outputs a list of the next
10 events on the user's calendar.
"""
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
service = discovery.build('calendar', 'v3', http=http)
now = datetime.datetime.utcnow().isoformat() + 'Z' # 'Z' indicates UTC time
print('Getting the upcoming 10 events')
eventsResult = service.events().list(
calendarId='primary', timeMin=now, maxResults=10, singleEvents=True,
orderBy='startTime').execute()
events = eventsResult.get('items', [])
if not events:
print('No upcoming events found.')
for event in events:
start = event['start'].get('dateTime')
# print(start)
start = start[:-9]
# print(start)
start = datetime.datetime.strptime(start,"%Y-%m-%dT%H:%M")
pretty_time = start.strftime("%I:%M")
pretty_date = start.strftime("%B %d, %Y")
print(event['summary'],"at",pretty_time,"on",pretty_date)
if __name__ == '__main__':
main()
below is the error which I'm getting. Not able to find the what is the mistake I made. please help.
below is the error which I'm getting. Not able to find the what is the mistake I made. please help.
Traceback (most recent call last):
File "/Users/mbhamidipati/python-practice/change/getevents.py", line 86, in <module>
main()
File "/Users/mbhamidipati/python-practice/change/getevents.py", line 75, in main
start = start[:-9]
TypeError: 'NoneType' object is not subscriptable
Thanks,
Sreman
It seems that the main thing you need to decide is how you're going to handle when
event['start'].get('dateTime') returns None
you can add a default value to .get('dateTime')
Something like this, assuming that you can get a value from another key, such as 'date'
start = event['start'].get('dateTime', event['start'].get('date'))
The code below works. I can send emails through the Gmail API, but it takes awhile to send 10 personalized emails, based on wall time (20-30 secs). Is there a way to optimize the code below to send emails faster? There are quota limitations.
Is the max number emails one can send, a 100 per day? There seems to be a difference between number of emails one can send and the number of receipts per email. This is the documentation I am sourcing: https://developers.google.com/apps-script/guides/services/quotas
I am using the consumer version.
Feature Consumer (gmail.com)
Calendar events created 5,000
Contacts created 1,000
Documents created 250
Email recipients per day 100*
Code:
import httplib2
import os
import oauth2client
from oauth2client import client, tools
import base64
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from apiclient import errors, discovery
import mimetypes
import pandas as pd
import textwrap
SCOPES = 'https://www.googleapis.com/auth/gmail.send'
CLIENT_SECRET_FILE = 'secret.json'
APPLICATION_NAME = 'AppName'
def get_credentials():
home_dir = os.path.expanduser('~')
credential_dir = os.path.join(home_dir, '.credentials')
if not os.path.exists(credential_dir):
os.makedirs(credential_dir)
credential_path = os.path.join(credential_dir,
'gmail-send.json')
store = oauth2client.file.Storage(credential_path)
credentials = store.get()
if not credentials or credentials.invalid:
flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
flow.user_agent = APPLICATION_NAME
credentials = tools.run_flow(flow, store)
print 'Storing credentials to ' + credential_path
return credentials
def SendMessage(sender, to, subject,message_text):
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
service = discovery.build('gmail', 'v1', http=http)
message1 = CreateMessageHtml(sender, to, subject, message_text)
result = SendMessageInternal(service, "me", message1)
return result
def CreateMessageHtml(sender, to, subject, message_text):
msg = MIMEText(message_text)
msg['Subject'] = subject
msg['From'] = sender
msg['To'] = to
return {'raw': base64.urlsafe_b64encode(msg.as_string())}
def SendMessageInternal(service, user_id, message):
try:
message = (service.users().messages().send(userId=user_id, body=message).execute())
print 'Message Id: %s' % message['id']
return message
except errors.HttpError, error:
print 'An error occurred: %s' % error
return "Error"
return "OK"
def main():
df = pd.read_csv('testdata.csv')
for index,row in df.iterrows():
to = row['Email']
sender = "sender"
subject = "subject"
dedent_text = '''Hello {}, \n
thank you for your question.'''.format(row['First'])
message_text = textwrap.dedent(dedent_text).strip()
SendMessage(sender, to, subject, message_text)
if __name__ == '__main__':
main()
Try caching the result of the service call, so that service is passed to SendMessage. This way, you don't take API call times to setup the API for each individual emails you send.
So at top of your main:
def main():
# Do once
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
service = discovery.build('gmail', 'v1', http=http)
df = pd.read_csv('testdata.csv')
for index,row in df.iterrows():
to = row['Email']
sender = "sender"
subject = "subject"
dedent_text = '''Hello {}, \n
thank you for your question.'''.format(row['First'])
message_text = textwrap.dedent(dedent_text).strip()
# service is is reused here for each message
SendMessage(service, sender, to, subject, message_text)
Also, if you need to send many messages, make sure you invoke one Python invokation per large batch, since starting up the interpreter and loading many packages can take a while each time.
I want to set up a watch request for a google calendar, using python
(without setting up a separate domain).
I imported the api client, and can successfully get the authenticated credentials, following the example: https://developers.google.com/google-apps/calendar/quickstart/python.
Then I set up a calendar service, and I am able to list, insert and delete events without any issue.
The problem I am having is when I perform a watch request so that I have a webhook from within python.
I receive the error:
"googleapiclient.errors.HttpError: https://www.googleapis.com/calendar/v3/calendars/primary/events/watch?alt=json returned "WebHook callback must be HTTPS:">"
Clearly I am missing something that needs to be setup so that calendar is satisfied with the webhook I am giving it.
Is it possible to do this from within python, without setting up a separate domain with https, and if so, how?
Minimum working example:
import httplib2
import os
from apiclient import discovery
from oauth2client import client
from oauth2client import tools
from oauth2client.file import Storage
import uuid
try:
import argparse
flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
flags = None
SCOPES = 'https://www.googleapis.com/auth/calendar'
CLIENT_SECRET_FILE = 'client_secret.json'
APPLICATION_NAME = 'Calendar API'
def get_credentials():
home_dir = os.path.expanduser('~')
credential_dir = os.path.join(home_dir, '.credentials')
if not os.path.exists(credential_dir):
os.makedirs(credential_dir)
credential_path = os.path.join(credential_dir,
'calendar-api.json')
store = Storage(credential_path)
credentials = store.get()
if not credentials or credentials.invalid:
flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
flow.user_agent = APPLICATION_NAME
if flags:
credentials = tools.run_flow(flow, store, flags)
else:
credentials = tools.run(flow, store)
print('Storing credentials to ' + credential_path)
return credentials
def main():
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
service = discovery.build('calendar', 'v3', http=http)
## TESTING callback receiver:
eventcollect = {
'id': str(uuid.uuid1()),
'type': "web_hook"
}
service.events().watch(calendarId='primary', body=eventcollect).execute()
if __name__ == '__main__':
main()
In the documentation of watch requests, there is a Required Properties part. In this part it is clearly stated that:
An address property string set to the URL that listens and responds to notifications for this notification channel. This is your Webhook callback URL, and it must use HTTPS.
I am sorry but I do not think there is a workaround for this.
In python how do i mark messages as 'read' as i parse it from gmail api ?
Also how do i save the values to the database after parsing?
This is the code so far to get the content of each message.
from __future__ import print_function
import httplib2
import os
import re
import MySQLdb
from email.utils import parsedate_tz,mktime_tz,formatdate
from requests.adapters import HTTPAdapter
import datetime
from datetime import date,timedelta
import time
from apiclient import discovery
import oauth2client
from oauth2client import client
from oauth2client import tools
import json
try:
import argparse
flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
flags = None
SCOPES = 'https://www.googleapis.com/auth/gmail.readonly'
CLIENT_SECRET_FILE = 'client_server.json'
APPLICATION_NAME = 'Gmail API Python Quickstart'
def get_credentials():
home_dir = os.path.expanduser('~')
credential_dir = os.path.join(home_dir, '.credentials')
if not os.path.exists(credential_dir):
os.makedirs(credential_dir)
credential_path = os.path.join(credential_dir,
'gmail-python-quickstart.json')
store = oauth2client.file.Storage(credential_path)
credentials = store.get()
if not credentials or credentials.invalid:
flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
flow.user_agent = APPLICATION_NAME
if flags:
credentials = tools.run_flow(flow, store, flags)
else: # Needed only for compatibility with Python 2.6
credentials = tools.run(flow, store)
print('Storing credentials to ' + credential_path)
return credentials
def main():
da=date.fromordinal(730920)
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
service = discovery.build('gmail', 'v1', http=http)
today=date.today()
print (today)
yesterday=today-timedelta(1)
print (yesterday)
response = service.users().messages().list(userId='me',q='in:inbox is:unread newer_than:1d').execute()
messages=[]
store=[]
message1=[]
test2=[]
da=[]
if 'messages' in response:
messages.extend(response['messages'])
fo = open("fooa.txt", "wb")
for i in range(len(messages)):
store=messages[i]['id']
message = service.users().messages().get(userId='me',id=store,format='metadata',metadataHeaders=['from','date']).execute()
fo.write(message['snippet'].encode('utf-8')+"")
From=message['payload']['headers'][0]['value']
fo.write(From+"");
da=message['payload']['headers'][1]['value']
fo.write(da+"\n");
for line in open("fooa.txt"):
print(line)
fo.close()
a=open("fooa.txt","r")
for wo in a:
match=re.findall(r':[\w]+',wo)
for word in match:
print(word.replace(':',' '))
db = MySQLdb.connect("localhost","testuser","mysql23","db1" )
cursor = db.cursor()
sql = """INSERT INTO customers((LeadName, CITY, SERVICE,CUSTOMER, MOBILE, EMAIL)
VALUES (, , , , )"""
try:
cursor.execute(sql)
db.commit()
except:
db.rollback()
db.close()
if __name__ == '__main__':
main()
Need help please!
You need to modify the message in a separate request, and remove the UNREAD-label.
POST https://www.googleapis.com/gmail/v1/users/me/messages/1533cb4d7dac1633/modify?access_token={ACCESS_TOKEN}
{
"removeLabelIds": [
"UNREAD"
]
}
If you're trying to mark the message as read, you will have to do something like:
gmail_service
.users()
.messages()
.modify(userId='me', id=message_id, body={'removeLabelIds': ['UNREAD']})
.execute()
What you can do with the Gmail API depends on what scope you've granted OAuth. Here's what you need to do:
Change SCOPES = 'https://www.googleapis.com/auth/gmail.readonly' to this:
SCOPES = [
'https://www.googleapis.com/auth/gmail.readonly',
'https://www.googleapis.com/auth/gmail.modify'
]
Explanation: since marking an email as "unread" is a message modification, you need the 'gmail.modify' scope in order to use this API request. To see the full list of scopes, refer to Choose Auth Scopes.
Delete your token.pickle file and rerun the script. You'll need to do this in order to reauthorize the OAuth app with the new scopes, which will then generate a new token.pickle file.
Here's the code to mark an email as read in Python:
service.users().messages().modify(userId='me', id=message['id'], body={
'removeLabelIds': ['UNREAD']
}).execute()
I am trying to download a spreadsheet file from my drive to my computer.
I am able to authenticate, get list of files and even get meta-data successfully.
But when I try to download the file, I get the following error :
downloading file starts
An error occurred: <HttpError 400 when requesting https://www.googleapis.com/dri
ve/v2/files/1vJetI_p8YEYiKvPVl0LtXGS5uIAx1eRGUupsXoh7UbI?alt=media returned "The
specified file does not support the requested alternate representation.">
downloading file ends
I couldn't get any such problem or question on SO and the other methods or solutions provided on SO for downloading the spreadsheet are outdated.Those have been deprecated by Google .
Here is the code, I am using to download the file :
import httplib2
import os
from apiclient import discovery
import oauth2client
from oauth2client import client
from oauth2client import tools
from apiclient import errors
from apiclient import http
try:
import argparse
flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
flags = None
#SCOPES = 'https://www.googleapis.com/auth/drive.metadata.readonly'
SCOPES = 'https://www.googleapis.com/auth/drive'
CLIENT_SECRET_FILE = 'client_secrets.json'
APPLICATION_NAME = 'Drive API Quickstart'
def get_credentials():
home_dir = os.path.expanduser('~')
credential_dir = os.path.join(home_dir, '.credentials')
if not os.path.exists(credential_dir):
os.makedirs(credential_dir)
credential_path = os.path.join(credential_dir,
'drive-quickstart.json')
store = oauth2client.file.Storage(credential_path)
credentials = store.get()
if not credentials or credentials.invalid:
flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
flow.user_agent = APPLICATION_NAME
if flags:
credentials = tools.run_flow(flow, store, flags)
else: # Needed only for compatability with Python 2.6
credentials = tools.run(flow, store)
print 'Storing credentials to ' + credential_path
return credentials
def main():
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
service = discovery.build('drive', 'v2', http=http)
file_id = '1vJetI_p8YEYiKvPVl0LtXGS5uIAx1eRGUupsXoh7UbI'
print "downloading file starts"
download_file(service, file_id)
print "downloading file ends "
def download_file(service, file_id):
local_fd = open("foo.csv", "w+")
request = service.files().get_media(fileId=file_id)
media_request = http.MediaIoBaseDownload(local_fd, request)
while True:
try:
download_progress, done = media_request.next_chunk()
except errors.HttpError, error:
print 'An error occurred: %s' % error
return
if download_progress:
print 'Download Progress: %d%%' % int(download_progress.progress() * 100)
if done:
print 'Download Complete'
return
if __name__ == '__main__':
main()
Google spreadsheets don't have media. Instead they have exportLinks. Get the file metadata, then look in the exportlinks and pick an appropriate URL.
This code worked for me. I only had to download client_secret.json from google developers dashboard and keep in the same directory as python script.
And in the list_of_lists variable I got a list with each row as list.
import gspread
import json
from oauth2client.client import SignedJwtAssertionCredentials
json_key = json.load(open('client_secret.json'))
scope = ['https://spreadsheets.google.com/feeds']
credentials = SignedJwtAssertionCredentials(json_key['client_email'], json_key['private_key'], scope)
gc = gspread.authorize(credentials)
sht1 = gc.open_by_key('<id_of_sheet>')
worksheet_list = sht1.worksheets()
worksheet = sht1.sheet1
list_of_lists = worksheet.get_all_values()
for row in list_of_lists :
print row