Develop Reference

how i get mails from gmail over imap with python

This is my script, the auth_string is right, i tryed this smtplib.SMTP('smtp.gmail.com:587') and its worked, imap is activ in my gmail settings, and yes, please help me :)
def command_to_url(command):
return '%s/%s' % (GOOGLE_ACCOUNTS_BASE_URL, command)
def call_refresh_token(client_id, client_secret, refresh_token):
params = {}
params['client_id'] = client_id
params['client_secret'] = client_secret
params['refresh_token'] = refresh_token
params['grant_type'] = 'refresh_token'
request_url = command_to_url('o/oauth2/token')
response = urllib.request.urlopen(request_url, urllib.parse.urlencode(params).encode('UTF-8')).read().decode('UTF-8')
return json.loads(response)
def refresh_authorization(google_client_id, google_client_secret, refresh_token):
response = call_refresh_token(google_client_id, google_client_secret, refresh_token)
return response['access_token'], response['expires_in']
def generate_oauth2_string(username, access_token, as_base64=False):
auth_string = 'user=%s\1auth=Bearer %s\1\1' % (username, access_token)
if as_base64:
auth_string = base64.b64encode(auth_string.encode('ascii')).decode('ascii')
return auth_string
def test_imap(user, auth_string):
imap_conn = imaplib.IMAP4_SSL('imap.gmail.com', port=993)
imap_conn.debug = 4
imap_conn.authenticate('XOAUTH2 ', lambda x: auth_string)
access_token, expires_in = refresh_authorization(GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET, GOOGLE_REFRESH_TOKEN)
auth_string = generate_oauth2_string('---------#gmail.com', access_token, as_base64=True)
test_imap('---------#gmail.com', auth_string)
response:
30:07.30 > b'KOHE1 AUTHENTICATE XOAUTH2 '
30:07.32 < b'+ '
30:07.32 write literal size 376
30:07.41 < b'+ eyJzdGF0dXMiOiI0MDAiLCJzY2hlbWVzIjoiQmVhcmVyIiwic2NvcGUiOiJodHRwczovL21haWwuZ29vZ2xlLmNvbS8ifQ=='
30:07.41 write literal size 376
30:07.48 < b'KOHE1 BAD Invalid SASL argument. q16mb213626858wmq'
30:07.48 BAD response: b'Invalid SASL argument. q16mb213626858wmq'
Traceback (most recent call last):
File "E:\path_to_script\mail_send.py", line 148, in <module>
test_imap('---------#gmail.com', auth_string)
File "E:\path_to_script\mail_send.py", line 78, in test_imap
imap_conn.authenticate('XOAUTH2 ', lambda x: auth_string)
File "C:\Users\username\AppData\Local\Programs\Python\Python37\lib\imaplib.py", line 428, in authenticate
typ, dat = self._simple_command('AUTHENTICATE', mech)
File "C:\Users\username\AppData\Local\Programs\Python\Python37\lib\imaplib.py", line 1196, in _simple_command
return self._command_complete(name, self._command(name, *args))
File "C:\Users\username\AppData\Local\Programs\Python\Python37\lib\imaplib.py", line 1027, in _command_complete
raise self.error('%s command error: %s %s' % (name, typ, data))
imaplib.error: AUTHENTICATE command error: BAD [b'Invalid SASL argument. q16mb213626858wmq']
I try this since 3 days and i dont anymore :[]

The solution is that my auth_string was as_base64 encoded, but imap wants the 'not encoded' auth_string
access_token, expires_in = refresh_authorization(GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET, GOOGLE_REFRESH_TOKEN)
auth_string = generate_oauth2_string('---------#gmail.com', access_token, as_base64=False)
test_imap('---------#gmail.com', auth_string)

Try step by step:
M = imaplib.IMAP4_SSL('imap.gmail.com')
try:
rv, data = M.login(EMAIL_ACCOUNT, EMAIL_PASSWORD)
except imaplib.IMAP4.error:
print "Login failed."
sys.exit(1)
This will test IMAP4_SSL with Gmail against your credentials (without OAUTH2, so verify this configuration to be enabled).
If that works, and XOAUTH2 does not, well clearly I must question your statement that the auth string is correct.
For example, the error you get back seems to refer to a scope of mail.google.com, while I would have expected imap.google.com. Is this correct?
Also, is the extra space after 'XOAUTH2' a typo, or is it in the original code? That might mess with the algo selection mechanism.
You might want to re-start afresh from the Google Python XOAUTH2 sample code and build up from there.

How to mock a method of a dependent class?

I'm trying to test a method (Creator.do_a_call) which calls another class's method (Requester.make_request) which makes external calls (all the code is below). So I want to mock Request.make_request to return a mocked out response object so it doesn't do any external calls, but when I do that it gives me an error (see below)
Here's the code:
Requester.py:
from requests import Response, Session, Request
class Requester:
url: str = ''
def __init__(self, url: str):
self.url = url
def make_request(self, path: str, method: str = 'get', body: dict = None, custom_headers: dict = None) -> Response:
print("make_request")
url = self.url + path
session = Session()
request = Request(method, url, json=body)
response = session.send(request.prepare())
return response
Creator.py
from Requester import Requester
from http import HTTPStatus as status
from requests import Response
class Creator:
def do_a_call(self, url) -> Response:
print("do_a_call")
requester = Requester(url)
response = requester.make_request(
"/",
"GET"
)
print(type(response))
if response.status_code != status.OK and response.status_code != status.ACCEPTED:
raise Exception(response.status_code, response.text)
return response
main.py
from Creator import Creator
print("main")
creator = Creator()
response = creator.do_a_call("https://google.com")
print(response)
test.py
import unittest
from unittest.mock import patch
from Creator import Creator
from requests import Session
import requests_mock
class CreatorTest(unittest.TestCase):
def mocked_make_request(self, two, three):
session = Session()
mocker = requests_mock.Mocker(session=session)
adapter = requests_mock.Adapter()
adapter.register_uri('POST', 'mock://test.com', text="Invalid policy number supplied", status_code=400)
return mocker.post("mock://test.com", status_code=400)
#patch('Requester.Requester.make_request')
def test_do_a_call_side_effect(self, make_request):
creator = Creator()
make_request.side_effect = self.mocked_make_request
response = creator.do_a_call("mock://test.com")
self.assertEqual(200, response.status_code)
#patch('Requester.Requester.make_request')
def test_do_a_call_return_value(self, make_request):
creator = Creator()
make_request.return_value = self.mocked_make_request("", "")
response = creator.do_a_call("mock://test.com")
self.assertEqual(200, response.status_code)
if __name__ == "__main__":
unittest.main()
In my research I've found two ways that should work - changing the return_value of the mock, or changing the side_effect of the mock, both of which give me the same error:
Traceback (most recent call last):
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/unittest/mock.py", line 1325, in patched
return func(*newargs, **newkeywargs)
File "test.py", line 19, in test_do_a_call_side_effect
response = creator.do_a_call("mock://test.com")
File "/Users/citrja/Documents/git/python_test/Creator.py", line 14, in do_a_call
if response.status_code != status.OK and response.status_code != status.ACCEPTED:
AttributeError: '_Matcher' object has no attribute 'status_code'

I've found out what was wrong! I Was not attaching the adapter to the session, I had thought this happened automagically but apparently not. Here's the updated mocked_make_request function:
def mocked_make_request(self, two, three):
session = Session()
mocker = requests_mock.Mocker(session=session)
adapter = requests_mock.Adapter()
adapter.register_uri('GET', 'mock://test.com', text="Invalid policy number supplied", status_code=400)
session.mount("mock://", adapter)
return session.get("mock://test.com")

Gnip issue - while creating a job -- urllib2.URLError

I am trying to create a job using Gnip Historical Powertrack API.
I am getting issue with the urllib..
import urllib2
import base64
import json
UN = '' # YOUR GNIP ACCOUNT EMAIL ID
PWD = ''
account = '' # YOUR GNIP ACCOUNT USER NAME
def get_json(data):
return json.loads(data.strip())
def post():
url = 'https://historical.gnip.com/accounts/' + account + '/jobs.json'
publisher = "twitter"
streamType = "track"
dataFormat = "activity-streams"
fromDate = "201510140630"
toDate = "201510140631"
jobTitle = "job30"
rules = '[{"value":"","tag":""}]'
jobString = '{"publisher":"' + publisher + '","streamType":"' + streamType + '","dataFormat":"' + dataFormat + '","fromDate":"' + fromDate + '","toDate":"' + toDate + '","title":"' + jobTitle + '","rules":' + rules + '}'
base64string = base64.encodestring('%s:%s' % (UN, PWD)).replace('\n', '')
req = urllib2.Request(url=url, data=jobString)
req.add_header('Content-type', 'application/json')
req.add_header("Authorization", "Basic %s" % base64string)
proxy = urllib2.ProxyHandler({'http': 'http://proxy:8080', 'https': 'https://proxy:8080'})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
try:
response = urllib2.urlopen(req)
the_page = response.read()
the_page = get_json(the_page)
print 'Job has been created.'
print 'Job UUID : ' + the_page['jobURL'].split("/")[-1].split(".")[0]
except urllib2.HTTPError as e:
print e.read()
if __name__=='__main__':
post()
this is the error I am getting :
Traceback (most recent call last):
File "gnip1.py", line 37, in <module>
post()
File "gnip1.py", line 28, in post
response = urllib2.urlopen(req)
File "/home/soundarya/anaconda-new-1/lib/python2.7/urllib2.py", line 154, in urlopen
return opener.open(url, data, timeout)
File "/home/soundarya/anaconda-new-1/lib/python2.7/urllib2.py", line 431, in open
response = self._open(req, data)
File "/home/soundarya/anaconda-new-1/lib/python2.7/urllib2.py", line 449, in _open
'_open', req)
File "/home/soundarya/anaconda-new-1/lib/python2.7/urllib2.py", line 409, in _call_chain
result = func(*args)
File "/home/soundarya/anaconda-new-1/lib/python2.7/urllib2.py", line 1240, in https_open
context=self._context)
File "/home/soundarya/anaconda-new-1/lib/python2.7/urllib2.py", line 1197, in do_open
raise URLError(err)
urllib2.URLError: <urlopen error [Errno -2] Name or service not known>
I even tried through the curl command:
When I tried running the below one in terminal, I am getting error - ServiceUsername is not valid.
curl -v -X POST -uname -d '{"title": "HPT_test_job","publisher": "Twitter","streamType":"track","dataFormat":"activity-streams","fromDate":"201401010000","toDate":"201401020000 ","rules":[{"value": "twitter_lang:en (Hillary Clinton OR Donald)","tag": "2014_01_01_snow"}]}' 'https://historical.gnip.com/accounts/account_name/jobs.json'
This is the exact output msg:
Error retrieving Job status: {u'serviceUsername': [u'is invalid']} -- Please verify your connection parameters and network connection *

Try this.. see if it helps
import urllib2
from urllib2.request import urlopen
u = urlopen ('http:// .........')

If you are using python 3.5 you should use the library urllib.request which is the newer version of urllib2. Notice however that this changes a few things in the code including print (which should be in parentheses) and the need to transform some of the string results into bytes. Here you can look at all the required changes in code adapted to python 3.5

Python perform Requests Function Call in Loop?

Script below is successful in pulling the right information from a single IP(url_ip). But after trying to build a looping process the requests call falls over with connection errors. (errors below)
*NOTE - sloppy code so be warned.
from lxml import html
import requests
import smtplib
# STATIC URL
#TODO PULL A LIST OF IP ADDRESSES AND BUILD THE URL FOR EACH SYSTEM
#IPs = ['192.168.3.152','192.168.3.194']
def crawler(url_ip):
global eqid, counter, serial
print "Starting Crawler Service for: " + url_ip
url = "http://" + url_ip + "/cgi-bin/dynamic/printer/config/reports/deviceinfo.html"
urleqid = "http://" + url_ip + "/cgi-bin/dynamic/topbar.html"
response = requests.get(url)
tree = html.fromstring(response.text)
counter = tree.xpath('//td[contains(p,"Count")]/following-sibling::td/p/text()')
serial = tree.xpath('//td[contains(p, "Serial")]/following-sibling::td/p/text()')
counter = counter[0].split(' ')[3]
serial = serial[0].split(' ')[3]
responseeqid = requests.get(urleqid)
treeequid = html.fromstring(responseeqid.text)
eqid = treeequid.xpath('//descendant-or-self::node()/child::b[contains(., "Location")]/text()')[1].split(' ')[-1]
print " -- equipment id found: " + eqid
print " -- count found: " + counter
print " -- serial found: " + serial
print "Stopping Crawler Service for: " + url_ip
return
def send_mail(eqid,counter,serial):
GMAIL_USERNAME = "removed"
GMAIL_PASSWORD = "removed"
recipient = "removed"
email_subject = "Test"
body_of_email = "Equipment ID = " + eqid + "<br>Total Meter Count = " + counter + "<br>Serial Number = " + serial + "<br><br>"
session = smtplib.SMTP('smtp.gmail.com', 587)
session.ehlo()
session.starttls()
session.login(GMAIL_USERNAME, GMAIL_PASSWORD)
headers = "\r\n".join(["from: " + GMAIL_USERNAME,
"subject: " + email_subject,
"to: " + recipient,
"mime-version: 1.0",
"content-type: text/html"])
# body_of_email can be plain text or html!
content = headers + "\r\n\r\n" + body_of_email
session.sendmail(GMAIL_USERNAME, recipient, content)
return
with open('iplist.txt') as fp:
for line in fp:
crawler(line);
#send_mail(eqid,counter,serial);
ERROR LOG:
Starting Crawler Service for: 192.168.3.152
Traceback (most recent call last):
File "getmeters.py", line 63, in <module>
crawler(ipstring);
File "getmeters.py", line 17, in crawler
response = requests.get(url)
File "/Library/Python/2.7/site-packages/requests/api.py", line 68, in get
return request('get', url, **kwargs)
File "/Library/Python/2.7/site-packages/requests/api.py", line 50, in request
response = session.request(method=method, url=url, **kwargs)
File "/Library/Python/2.7/site-packages/requests/sessions.py", line 464, in request
resp = self.send(prep, **send_kwargs)
File "/Library/Python/2.7/site-packages/requests/sessions.py", line 576, in send
r = adapter.send(request, **kwargs)
File "/Library/Python/2.7/site-packages/requests/adapters.py", line 415, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', gaierror(8, 'nodename nor servname provided, or not known'))
I thought it was due to the value "line" being processed as a list object and not a string, so I converted to str(line) and that failed as well.

I suspect that you have line endings (\n) at the end of the lines in the files, and you may need to strip those off. otherwise your URL becomes something like
http://192.168.3.152
/cgi-bin/dynamic/printer/config/reports/deviceinfo.html"
instead of the intended
http://192.168.3.152/cgi-bin/dynamic/printer/config/reports/deviceinfo.html"

Converting python2 code to python3 problems

So I have been trying too convert an omegle bot, which was written in python2, to python3. This is the original code: https://gist.github.com/thefinn93/1543082
Now this is my code:
import requests
import sys
import json
import urllib
import random
import time
server = b"odo-bucket.omegle.com"
debug_log = False # Set to FALSE to disable excessive messages
config = {'verbose': open("/dev/null","w")}
headers = {}
headers['Referer'] = b'http://odo-bucket.omegle.com/'
headers['Connection'] = b'keep-alive'
headers['User-Agent'] = b'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.2 (KHTML, like Gecko) Ubuntu/11.10 Chromium/15.0.874.106 Chrome/15.0.874.106 Safari/535.2'
headers['Content-type'] = b'application/x-www-form-urlencoded; charset=UTF-8'
headers['Accept'] = b'application/json'
headers['Accept-Encoding'] = b'gzip,deflate,sdch'
headers['Accept-Language'] = b'en-US'
headers['Accept-Charset'] = b'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
if debug_log:
config['verbose'] = debug_log
def debug(msg):
if debug_log:
print("DEBUG: " + str(msg))
debug_log.write(str(msg) + "\n")
def getcookies():
r = requests.get(b"http://" + server + b"/")
debug(r.cookies)
return(r.cookies)
def start():
r = requests.request(b"POST", b"http://" + server + b"/start?rcs=1&spid=", data=b"rcs=1&spid=", headers=headers)
omegle_id = r.content.strip(b"\"")
print("Got ID: " + str(omegle_id))
cookies = getcookies()
event(omegle_id, cookies)
def send(omegle_id, cookies, msg):
r = requests.request(b"POST","http://" + server + "/send", data="msg=" + urllib.quote_plus(msg) + "&id=" + omegle_id, headers=headers, cookies=cookies)
if r.content == "win":
print("You: " + msg)
else:
print("Error sending message, check the log")
debug(r.content)
def event(omegle_id, cookies):
captcha = False
next = False
r = requests.request(b"POST",b"http://" + server + b"/events",data=b"id=" + omegle_id, cookies=cookies, headers=headers)
try:
parsed = json.loads(r.content)
for e in parsed:
if e[0] == "waiting":
print("Waiting for a connection...")
elif e[0] == "count":
print("There are " + str(e[1]) + " people connected to Omegle")
elif e[0] == "connected":
print("Connection established!")
send(omegle_id, cookies, "HI I just want to talk ;_;")
elif e[0] == "typing":
print("Stranger is typing...")
elif e[0] == "stoppedTyping":
print ("Stranger stopped typing")
elif e[0] == "gotMessage":
print("Stranger: " + e[1])
try:
cat=""
time.sleep(random.randint(1,5))
i_r=random.randint(1,8)
if i_r==1:
cat="that's cute :3"
elif i_r==2:
cat="yeah, guess your right.."
elif i_r==3:
cat="yeah, tell me something about yourself!!"
elif i_r==4:
cat="what's up"
elif i_r==5:
cat="me too"
else:
time.sleep(random.randint(3,9))
send(omegle_id, cookies, "I really have to tell you something...")
time.sleep(random.randint(3,9))
cat="I love you."
send(omegle_id, cookies, cat)
except:
debug("Send errors!")
elif e[0] == "strangerDisconnected":
print("Stranger Disconnected")
next = True
elif e[0] == "suggestSpyee":
print ("Omegle thinks you should be a spy. Fuck omegle.")
elif e[0] == "recaptchaRequired":
print("Omegle think's you're a bot (now where would it get a silly idea like that?). Fuckin omegle. Recaptcha code: " + e[1])
captcha = True
except:
print("Derka derka derka")
if next:
print("Reconnecting...")
start()
elif not captcha:
event(omegle_id, cookies)
start()
The error I get is:
Traceback (most recent call last):
File "p3.py", line 124, in <module>
start()
File "p3.py", line 46, in start
r = requests.request(b"POST", b"http://" + server + b"/start?rcs=1&spid=", data=b"rcs=1&spid=", headers=headers)
File "/usr/lib/python3.4/site-packages/requests/api.py", line 44, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/lib/python3.4/site-packages/requests/sessions.py", line 456, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3.4/site-packages/requests/sessions.py", line 553, in send
adapter = self.get_adapter(url=request.url)
File "/usr/lib/python3.4/site-packages/requests/sessions.py", line 608, in get_adapter
raise InvalidSchema("No connection adapters were found for '%s'" % url)
requests.exceptions.InvalidSchema: No connection adapters were found for 'b'http://odo-bucket.omegle.com/start?rcs=1&spid=''
I didn't really understand what would fix this error, nor what the problem really is, even after looking it up.
UPDATE:
Now after removing all the b's I get the following error:
Traceback (most recent call last):
File "p3.py", line 124, in <module>
start()
File "p3.py", line 47, in start
omegle_id = r.content.strip("\"")
TypeError: Type str doesn't support the buffer API
UPDATE 2:
After putting the b back to r.content, I get the following error message:
Traceback (most recent call last):
File "p3.py", line 124, in <module>
start()
File "p3.py", line 50, in start
event(omegle_id, cookies)
File "p3.py", line 63, in event
r = requests.request("POST","http://" + server + "/events",data="id=" + omegle_id, cookies=cookies, headers=headers)
TypeError: Can't convert 'bytes' object to str implicitly
UPDATE 3:
Everytime I try to start it excepts "Derka derka", what could be causing this (It wasn't like that with python2).

requests takes strings, not bytes values for the URL.
Because your URLs are bytes values, requests is converting them to strings with str(), and the resulting string contains the characters b' at the start. That's no a valid scheme like http:// or https://.
The majority of your bytestrings should really be regular strings instead; only the content.strip() call deals with actual bytes.
The headers will be encoded for you, for example. Don't even set the Content-Type header; requests will take care of that for you if you pass in a dictionary (using string keys and values) to the data keyword argument.
You shouldn't set the Connection header either; leave connection management to requests as well.