Develop Reference

Python IMAP4 Don't Mark messages as read

I have a Python script to move messages with a certain subject. The messages get marked as read and I don't want them to be marked as read. What part of the script marks them as read and how do I make it not mark as read?
Also, I'm not too sure what I am doing yet, so if there is any redundant code or errors please let me know.
import getpass
from Crypto.Hash import MD5
import sys
import imaplib
import email
import re
password = getpass.getpass()
match = "redacted"
username = "redacted"
dest = "000"
pattern_uid = re.compile('\d+ \(UID (?P<uid>\d+)\)')
def md5(message):
hash = MD5.new()
hash.update(message)
return hash.hexdigest()
md5 = md5(password)
if md5 == match:
pass
else:
print "Mismatch"
sys.exit()
M = imaplib.IMAP4_SSL("mail.redacted.com", 993)
M.login(username, password)
M.select()
typ, data = M.search(None, 'ALL')
M.select('Inbox')
msgs = M.search(None, 'ALL')[1]
num_messages = len(msgs[0].split())
num_messages += 1
def parse_uid(data):
match = pattern_uid.match(data)
return match.group('uid')
for i in range(1, num_messages):
try:
typ, msg_data = M.fetch(str(i), '(RFC822)')
except:
pass
for response_part in msg_data:
if isinstance(response_part, tuple):
UID = M.fetch(str(i),'UID')
UID = UID[1]
try:
UID = parse_uid(UID[0])
except:
pass
msg = email.message_from_string(response_part[1])
for header in [ 'subject' ]:
if msg[header] == "Redacted":
result = M.uid('COPY', UID, dest)
if result[0] == 'OK':
mov, data = M.uid('STORE', UID, '+FLAGS', '(\Deleted)')
M.expunge()
M.close()
M.logout()

typ, msg_data = M.fetch(str(i), '(RFC822)')
Fetching a message body marks it as read. You'll want to use BODY.PEEK[].
Although, I don't know why you're fetching the whole message just to copy it. Why don't you just fetch the headers? Use BODY.PEEK[HEADERS].

Ldif to shadow file conversion

I have been trying to do some password cracking using L0phtcrack. I have a LDIF file that cannot be imported in to l0phtcrack as it is. lophtcrack does not also import from a csv file. The only option that I have using lophtcrack is to import from a Unix shadow file. Has anyone tried out converting a ldif file to a shadow file format in python ? If yes, would you mind sharing the script here.
Thanks
UPDATE - I made a few edits to code that i found online and got it to work. I now have a shadow file from my LDIF extract.
#!/usr/bin/env python
"""
Read in a ldap/ldif file and split out a unix-formated password file to run in
john the ripper for password auditing.
Typical usage (on the ldap server):
Dump the ldap database
slapcat > ldif.out
Convert it to passwd format
ldap-passwd-dump.py ldif.out passwd.out
Run john to crack passwords
john passwd.out
Aaron Peterson <aaron#midnightresearch.com>
"""
import sys, base64, re, os
class Results:
"""
Keep all user results here
"""
def __init__(self):
self.users = []
def addUser(self, user):
# Check for duplicates
dupe = 0
for u in self.users:
if u.uid == user.uid:
dupe = 1
break
if not dupe:
print " [*] Adding new user [%s, %s] to results" % (user.cn, user.uid)
self.users.append(user)
else:
print " [*] Not adding duplicate user [%s]" % user.cn
class User(list):
def __init__(self, hash=None, base64=None, password=None, cn=None, uid=None):
self.hash = hash
self.uid = uid
self.base64 = base64
self.password = password
self.cn = cn
list.__init__(self)
class LDIFCrack:
def main(self):
# Open file
f = open(self.ldif, "r")
# Load in the first user
user = User()
isInGroup=0
# Load lines into a "user"
for line in f:
if re.compile(r"^\s*$").search(line):
# Only append the old user if it's in the right group, and has a password set
if isInGroup and user.hash:
self.results.addUser(user)
# Reset user and counter
user = User()
isInGroup=0
# Make sure we test the right groups
if re.compile(self.groupMatch).search(line):
isInGroup=1
# Pull out the password
match = re.compile(r"userpassword: (.*)$").search(line)
if match:
user.hash = match.group(1)
# uid
match = re.compile(r"uid: (.*)$").search(line)
if match:
user.uid= match.group(1)
# Grab the common name
matchCn = re.compile(r"cn: (.*)$").search(line)
if matchCn:
user.cn = matchCn.group(1)
def printPasswd(self, file):
f = open(file, "w")
for user in self.results.users:
line = "%s:%s:::%s" % (user.uid, user.hash, user.cn)
f.write(line + "\n")
print " [*] %s" % line
f.close()
print " [*] Wrote [%s] password lines to [%s] " % (len(self.results.users), file)
def __init__(self, ldif, groupMatch):
self.ldif = ldif
self.results = Results()
self.groupMatch = groupMatch
self.main()
if __name__ == "__main__":
if len(sys.argv) < 3:
print "\nusage: %s <ldif file> <output password file> [<user matchString>]" % sys.argv[0]
print " example: %s ldif.out passwd.txt \"^ou: MyGroup\"" % sys.argv[0]
print " (matchString default is \"objectClass: posixAccount\")\n"
sys.exit(1)
ldif = sys.argv[1]
passwdFile = sys.argv[2]
if not os.path.exists(ldif):
print " [!] LDIF Input file [%s] does not exist..." % ldif
sys.exit(1)
if os.path.exists(passwdFile):
print " [!] Won't overwrite existing passwd file [%s]" % passwdFile
sys.exit(1)
# Will match the user against this group before cracking it if it's set
if len(sys.argv) == 4:
groupMatch = sys.argv[3]
else:
groupMatch = "objectClass: posixAccount"
ldifcrack = LDIFCrack(ldif, groupMatch)
ldifcrack.printPasswd(passwdFile)
print " [*] Done"

import python with __main__ method

I have a python script that have __main__ statement and took all values parametric.
I want to import and use it in my own script.
Actually I can import but don't know how to use it.
As you see below, __main__ is a bit complicated and rewriting it will take time because I even don't know what does most of code mean.
Want to know is there any way to import and use the code as a function?
import os
import sys
import time
import base64
from urllib2 import urlopen
from urllib2 import Request
from urllib2 import HTTPError
from urllib import urlencode
from urllib import quote
from exceptions import Exception
from email.mime.multipart import MIMEMultipart
from email.mime.base import MIMEBase
from email.mime.application import MIMEApplication
from email.encoders import encode_noop
from api_util import json2python, python2json
class MalformedResponse(Exception):
pass
class RequestError(Exception):
pass
class Client(object):
default_url = 'http://nova.astrometry.net/api/'
def __init__(self,
apiurl = default_url):
self.session = None
self.apiurl = apiurl
def get_url(self, service):
return self.apiurl + service
def send_request(self, service, args={}, file_args=None):
'''
service: string
args: dict
'''
if self.session is not None:
args.update({ 'session' : self.session })
print 'Python:', args
json = python2json(args)
print 'Sending json:', json
url = self.get_url(service)
print 'Sending to URL:', url
# If we're sending a file, format a multipart/form-data
if file_args is not None:
m1 = MIMEBase('text', 'plain')
m1.add_header('Content-disposition', 'form-data; name="request-json"')
m1.set_payload(json)
m2 = MIMEApplication(file_args[1],'octet-stream',encode_noop)
m2.add_header('Content-disposition',
'form-data; name="file"; filename="%s"' % file_args[0])
#msg.add_header('Content-Disposition', 'attachment',
# filename='bud.gif')
#msg.add_header('Content-Disposition', 'attachment',
# filename=('iso-8859-1', '', 'FuSballer.ppt'))
mp = MIMEMultipart('form-data', None, [m1, m2])
# Makie a custom generator to format it the way we need.
from cStringIO import StringIO
from email.generator import Generator
class MyGenerator(Generator):
def __init__(self, fp, root=True):
Generator.__init__(self, fp, mangle_from_=False,
maxheaderlen=0)
self.root = root
def _write_headers(self, msg):
# We don't want to write the top-level headers;
# they go into Request(headers) instead.
if self.root:
return
# We need to use \r\n line-terminator, but Generator
# doesn't provide the flexibility to override, so we
# have to copy-n-paste-n-modify.
for h, v in msg.items():
print >> self._fp, ('%s: %s\r\n' % (h,v)),
# A blank line always separates headers from body
print >> self._fp, '\r\n',
# The _write_multipart method calls "clone" for the
# subparts. We hijack that, setting root=False
def clone(self, fp):
return MyGenerator(fp, root=False)
fp = StringIO()
g = MyGenerator(fp)
g.flatten(mp)
data = fp.getvalue()
headers = {'Content-type': mp.get('Content-type')}
if False:
print 'Sending headers:'
print ' ', headers
print 'Sending data:'
print data[:1024].replace('\n', '\\n\n').replace('\r', '\\r')
if len(data) > 1024:
print '...'
print data[-256:].replace('\n', '\\n\n').replace('\r', '\\r')
print
else:
# Else send x-www-form-encoded
data = {'request-json': json}
print 'Sending form data:', data
data = urlencode(data)
print 'Sending data:', data
headers = {}
request = Request(url=url, headers=headers, data=data)
try:
f = urlopen(request)
txt = f.read()
print 'Got json:', txt
result = json2python(txt)
print 'Got result:', result
stat = result.get('status')
print 'Got status:', stat
if stat == 'error':
errstr = result.get('errormessage', '(none)')
raise RequestError('server error message: ' + errstr)
return result
except HTTPError, e:
print 'HTTPError', e
txt = e.read()
open('err.html', 'wb').write(txt)
print 'Wrote error text to err.html'
def login(self, apikey):
args = { 'apikey' : apikey }
result = self.send_request('login', args)
sess = result.get('session')
print 'Got session:', sess
if not sess:
raise RequestError('no session in result')
self.session = sess
def _get_upload_args(self, **kwargs):
args = {}
for key,default,typ in [('allow_commercial_use', 'd', str),
('allow_modifications', 'd', str),
('publicly_visible', 'y', str),
('scale_units', None, str),
('scale_type', None, str),
('scale_lower', None, float),
('scale_upper', None, float),
('scale_est', None, float),
('scale_err', None, float),
('center_ra', None, float),
('center_dec', None, float),
('radius', None, float),
('downsample_factor', None, int),
('tweak_order', None, int),
('crpix_center', None, bool),
# image_width, image_height
]:
if key in kwargs:
val = kwargs.pop(key)
val = typ(val)
args.update({key: val})
elif default is not None:
args.update({key: default})
print 'Upload args:', args
return args
def url_upload(self, url, **kwargs):
args = dict(url=url)
args.update(self._get_upload_args(**kwargs))
result = self.send_request('url_upload', args)
return result
def upload(self, fn, **kwargs):
args = self._get_upload_args(**kwargs)
try:
f = open(fn, 'rb')
result = self.send_request('upload', args, (fn, f.read()))
return result
except IOError:
print 'File %s does not exist' % fn
raise
def submission_images(self, subid):
result = self.send_request('submission_images', {'subid':subid})
return result.get('image_ids')
def overlay_plot(self, service, outfn, wcsfn, wcsext=0):
from astrometry.util import util as anutil
wcs = anutil.Tan(wcsfn, wcsext)
params = dict(crval1 = wcs.crval[0], crval2 = wcs.crval[1],
crpix1 = wcs.crpix[0], crpix2 = wcs.crpix[1],
cd11 = wcs.cd[0], cd12 = wcs.cd[1],
cd21 = wcs.cd[2], cd22 = wcs.cd[3],
imagew = wcs.imagew, imageh = wcs.imageh)
result = self.send_request(service, {'wcs':params})
print 'Result status:', result['status']
plotdata = result['plot']
plotdata = base64.b64decode(plotdata)
open(outfn, 'wb').write(plotdata)
print 'Wrote', outfn
def sdss_plot(self, outfn, wcsfn, wcsext=0):
return self.overlay_plot('sdss_image_for_wcs', outfn,
wcsfn, wcsext)
def galex_plot(self, outfn, wcsfn, wcsext=0):
return self.overlay_plot('galex_image_for_wcs', outfn,
wcsfn, wcsext)
def myjobs(self):
result = self.send_request('myjobs/')
return result['jobs']
def job_status(self, job_id, justdict=False):
result = self.send_request('jobs/%s' % job_id)
if justdict:
return result
stat = result.get('status')
if stat == 'success':
result = self.send_request('jobs/%s/calibration' % job_id)
print 'Calibration:', result
result = self.send_request('jobs/%s/tags' % job_id)
print 'Tags:', result
result = self.send_request('jobs/%s/machine_tags' % job_id)
print 'Machine Tags:', result
result = self.send_request('jobs/%s/objects_in_field' % job_id)
print 'Objects in field:', result
result = self.send_request('jobs/%s/annotations' % job_id)
print 'Annotations:', result
result = self.send_request('jobs/%s/info' % job_id)
print 'Calibration:', result
return stat
def sub_status(self, sub_id, justdict=False):
result = self.send_request('submissions/%s' % sub_id)
if justdict:
return result
return result.get('status')
def jobs_by_tag(self, tag, exact):
exact_option = 'exact=yes' if exact else ''
result = self.send_request(
'jobs_by_tag?query=%s&%s' % (quote(tag.strip()), exact_option),
{},
)
return result
if __name__ == '__main__':
import optparse
parser = optparse.OptionParser()
parser.add_option('--server', dest='server', default=Client.default_url,
help='Set server base URL (eg, %default)')
parser.add_option('--apikey', '-k', dest='apikey',
help='API key for Astrometry.net web service; if not given will check AN_API_KEY environment variable')
parser.add_option('--upload', '-u', dest='upload', help='Upload a file')
parser.add_option('--wait', '-w', dest='wait', action='store_true', help='After submitting, monitor job status')
parser.add_option('--wcs', dest='wcs', help='Download resulting wcs.fits file, saving to given filename; implies --wait if --urlupload or --upload')
parser.add_option('--kmz', dest='kmz', help='Download resulting kmz file, saving to given filename; implies --wait if --urlupload or --upload')
parser.add_option('--urlupload', '-U', dest='upload_url', help='Upload a file at specified url')
parser.add_option('--scale-units', dest='scale_units',
choices=('arcsecperpix', 'arcminwidth', 'degwidth', 'focalmm'), help='Units for scale estimate')
#parser.add_option('--scale-type', dest='scale_type',
# choices=('ul', 'ev'), help='Scale bounds: lower/upper or estimate/error')
parser.add_option('--scale-lower', dest='scale_lower', type=float, help='Scale lower-bound')
parser.add_option('--scale-upper', dest='scale_upper', type=float, help='Scale upper-bound')
parser.add_option('--scale-est', dest='scale_est', type=float, help='Scale estimate')
parser.add_option('--scale-err', dest='scale_err', type=float, help='Scale estimate error (in PERCENT), eg "10" if you estimate can be off by 10%')
parser.add_option('--ra', dest='center_ra', type=float, help='RA center')
parser.add_option('--dec', dest='center_dec', type=float, help='Dec center')
parser.add_option('--radius', dest='radius', type=float, help='Search radius around RA,Dec center')
parser.add_option('--downsample', dest='downsample_factor', type=int, help='Downsample image by this factor')
parser.add_option('--parity', dest='parity', choices=('0','1'), help='Parity (flip) of image')
parser.add_option('--tweak-order', dest='tweak_order', type=int, help='SIP distortion order (default: 2)')
parser.add_option('--crpix-center', dest='crpix_center', action='store_true', default=None, help='Set reference point to center of image?')
parser.add_option('--sdss', dest='sdss_wcs', nargs=2, help='Plot SDSS image for the given WCS file; write plot to given PNG filename')
parser.add_option('--galex', dest='galex_wcs', nargs=2, help='Plot GALEX image for the given WCS file; write plot to given PNG filename')
parser.add_option('--substatus', '-s', dest='sub_id', help='Get status of a submission')
parser.add_option('--jobstatus', '-j', dest='job_id', help='Get status of a job')
parser.add_option('--jobs', '-J', dest='myjobs', action='store_true', help='Get all my jobs')
parser.add_option('--jobsbyexacttag', '-T', dest='jobs_by_exact_tag', help='Get a list of jobs associated with a given tag--exact match')
parser.add_option('--jobsbytag', '-t', dest='jobs_by_tag', help='Get a list of jobs associated with a given tag')
parser.add_option( '--private', '-p',
dest='public',
action='store_const',
const='n',
default='y',
help='Hide this submission from other users')
parser.add_option('--allow_mod_sa','-m',
dest='allow_mod',
action='store_const',
const='sa',
default='d',
help='Select license to allow derivative works of submission, but only if shared under same conditions of original license')
parser.add_option('--no_mod','-M',
dest='allow_mod',
action='store_const',
const='n',
default='d',
help='Select license to disallow derivative works of submission')
parser.add_option('--no_commercial','-c',
dest='allow_commercial',
action='store_const',
const='n',
default='d',
help='Select license to disallow commercial use of submission')
opt,args = parser.parse_args()
if opt.apikey is None:
# try the environment
opt.apikey = os.environ.get('AN_API_KEY', None)
if opt.apikey is None:
parser.print_help()
print
print 'You must either specify --apikey or set AN_API_KEY'
sys.exit(-1)
args = {}
args['apiurl'] = opt.server
c = Client(**args)
c.login(opt.apikey)
if opt.upload or opt.upload_url:
if opt.wcs or opt.kmz:
opt.wait = True
kwargs = dict(
allow_commercial_use=opt.allow_commercial,
allow_modifications=opt.allow_mod,
publicly_visible=opt.public)
if opt.scale_lower and opt.scale_upper:
kwargs.update(scale_lower=opt.scale_lower,
scale_upper=opt.scale_upper,
scale_type='ul')
elif opt.scale_est and opt.scale_err:
kwargs.update(scale_est=opt.scale_est,
scale_err=opt.scale_err,
scale_type='ev')
elif opt.scale_lower or opt.scale_upper:
kwargs.update(scale_type='ul')
if opt.scale_lower:
kwargs.update(scale_lower=opt.scale_lower)
if opt.scale_upper:
kwargs.update(scale_upper=opt.scale_upper)
for key in ['scale_units', 'center_ra', 'center_dec', 'radius',
'downsample_factor', 'tweak_order', 'crpix_center',]:
if getattr(opt, key) is not None:
kwargs[key] = getattr(opt, key)
if opt.parity is not None:
kwargs.update(parity=int(opt.parity))
if opt.upload:
upres = c.upload(opt.upload, **kwargs)
if opt.upload_url:
upres = c.url_upload(opt.upload_url, **kwargs)
stat = upres['status']
if stat != 'success':
print 'Upload failed: status', stat
print upres
sys.exit(-1)
opt.sub_id = upres['subid']
if opt.wait:
if opt.job_id is None:
if opt.sub_id is None:
print "Can't --wait without a submission id or job id!"
sys.exit(-1)
while True:
stat = c.sub_status(opt.sub_id, justdict=True)
print 'Got status:', stat
jobs = stat.get('jobs', [])
if len(jobs):
for j in jobs:
if j is not None:
break
if j is not None:
print 'Selecting job id', j
opt.job_id = j
break
time.sleep(5)
success = False
while True:
stat = c.job_status(opt.job_id, justdict=True)
print 'Got job status:', stat
if stat.get('status','') in ['success']:
success = (stat['status'] == 'success')
break
time.sleep(5)
if success:
c.job_status(opt.job_id)
# result = c.send_request('jobs/%s/calibration' % opt.job_id)
# print 'Calibration:', result
# result = c.send_request('jobs/%s/tags' % opt.job_id)
# print 'Tags:', result
# result = c.send_request('jobs/%s/machine_tags' % opt.job_id)
# print 'Machine Tags:', result
# result = c.send_request('jobs/%s/objects_in_field' % opt.job_id)
# print 'Objects in field:', result
#result = c.send_request('jobs/%s/annotations' % opt.job_id)
#print 'Annotations:', result
retrieveurls = []
if opt.wcs:
# We don't need the API for this, just construct URL
url = opt.server.replace('/api/', '/wcs_file/%i' % opt.job_id)
retrieveurls.append((url, opt.wcs))
if opt.kmz:
url = opt.server.replace('/api/', '/kml_file/%i/' % opt.job_id)
retrieveurls.append((url, opt.kmz))
for url,fn in retrieveurls:
print 'Retrieving file from', url, 'to', fn
f = urlopen(url)
txt = f.read()
w = open(fn, 'wb')
w.write(txt)
w.close()
print 'Wrote to', fn
opt.job_id = None
opt.sub_id = None
if opt.sdss_wcs:
(wcsfn, outfn) = opt.sdss_wcs
c.sdss_plot(outfn, wcsfn)
if opt.galex_wcs:
(wcsfn, outfn) = opt.galex_wcs
c.galex_plot(outfn, wcsfn)
if opt.sub_id:
print c.sub_status(opt.sub_id)
if opt.job_id:
print c.job_status(opt.job_id)
#result = c.send_request('jobs/%s/annotations' % opt.job_id)
#print 'Annotations:', result
if opt.jobs_by_tag:
tag = opt.jobs_by_tag
print c.jobs_by_tag(tag, None)
if opt.jobs_by_exact_tag:
tag = opt.jobs_by_exact_tag
print c.jobs_by_tag(tag, 'yes')
if opt.myjobs:
jobs = c.myjobs()
print jobs
#print c.submission_images(1)

No, there is no clean way to do so. When the module is being imported, it's code is executed and all global variables are set as attributes to the module object. So if part of the code is not executed at all (is guarded by __main__ condition) there is no clean way to get access to that code. You can however run code of this module with substituted __name__ but that's very hackish.
You should refactor this module and move whole __main__ part into a method and call it like this:
def main():
do_everything()
if __name__ == '__main__':
main()
This way consumer apps will be able to run code without having to run it in a separate process.

Use the runpy module in the Python 3 Standard Library
See that data can be passed to and from the called script
# top.py
import runpy
import sys
sys.argv += ["another parameter"]
module_globals_dict = runpy.run_path("other_script.py",
init_globals = globals(), run_name="__main__")
print(module_globals_dict["return_value"])
# other_script.py
# Note we did not load sys module, it gets passed to this script
script_name = sys.argv[0]
print(f"Script {script_name} loaded")
if __name__ == "__main__":
params = sys.argv[1:]
print(f"Script {script_name} run with params: {params}")
return_value = f"{script_name} Done"

by what your saying you want to call a function in the script that is importing the module so try:
import __main__
__main__.myfunc()

python calling apis events and printing randomly

I have a interval of 60 and wanted to print 6 events every 1 minutes. But it prints 11,12 and 13 events randomly every 1 minutes. Why is that so ? Is it because of my codes or what other factors can cause this ?
My code is -
import logging
import httplib
import simplejson as json
import socket
import time
import datetime
import urllib2
import sys
import xml.dom.minidom
from bs4 import BeautifulSoup as soup
SCHEME = """<scheme>
<title>testingCurrentWeatherSG</title>
<description>Get data from forecast.</description>
<use_external_validation>true</use_external_validation>
<streaming_mode>simple</streaming_mode>
<endpoint>
<args>
<arg name="intervalone">
<title>Intervalone</title>
<description>How long to refresh this query?</description>
</arg>
</args>
</endpoint>
</scheme>
"""
def do_scheme():
print SCHEME
## Utility functions
def fahrenheit(fahren):
return (fahren-32) * 5.0/9.0
def get_percent(num):
return num * 100.
## Responses
def get_response(conn, url):
try:
conn.request('GET', url)
result = conn.getresponse()
data = result.read()
return json.loads(data)
except socket.timeout:
return None
## Printing
def print_forecast(name, di):
# Print the forcast from 'di', for location 'name'
# name is the name of the location, di is the api response
psi_avg=20
current = di['currently']
for key, value in sorted(current.iteritems()):
if key in ['cloudCover', 'icon', 'ozone', 'precipIntensity', # time
'precipProbability', 'precipType', 'pressure', 'summary',
'visibility', 'windBearing', 'windSpeed']:
print '{0} : {1}'.format(key, value)
elif key in ['temperature', 'dewPoint']:
print '%s: %.2f' % (key, fahrenheit(value))
elif key == 'humidity':
print '%s: %.2f' % (key, get_percent(value))
print 'psiAverage : ' + str(psi_avg)
print 'latitude : ' + str(di['latitude'])
print 'longitude : ' + str(di['longitude'])
print 'location : ' + str(name)
print
def weather_Connection(intervalone):
host = 'api.forecast.io'
conn = httplib.HTTPSConnection(host, timeout=60) # adjust timeout as desired
try:
urlnyp = '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.37871,103.848808'
conn.request('GET', urlnyp)
resultnyp = conn.getresponse()
contentnyp = resultnyp.read()
except socket.timeout:
print 'socket timeout'
return
# the locations and urls for the api calls
urls = {
'Choa Chu Kang': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.394557,103.746396',
'Kallang': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.311469,103.871399',
'Jurong West': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.352008,103.698599',
'Redhill': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.289732,103.81675',
'Tampines': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.353092,103.945229',
'Yishun': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.429463,103.84022',
}
responses = {}
for i, (name, url) in enumerate(sorted(urls.iteritems())):
response = get_response(conn, url)
if not response:
print 'socket timeout on url#%d: %s' % (i, url)
return
responses[name] = response
conn.close()
# print the forecast
for name, data in responses.iteritems():
print_forecast(name, data)
def get_config():
#Read XML Configuration data passed from splunkd on stdin
config = {}
try:
# read everything from stdin
config_str = sys.stdin.read()
# parse the config XML
doc = xml.dom.minidom.parseString(config_str)
root = doc.documentElement
conf_node = root.getElementsByTagName("configuration")[0]
if conf_node:
logging.debug("XML: found configuration")
stanza = conf_node.getElementsByTagName("stanza")[0]
if stanza:
stanza_name = stanza.getAttribute("name")
if stanza_name:
logging.debug("XML: found stanza " + stanza_name)
config["name"] = stanza_name
params = stanza.getElementsByTagName("param")
for param in params:
param_name = param.getAttribute("name")
logging.debug("XML: found param '%s'" % param_name)
if param_name and param.firstChild and \
param.firstChild.nodeType == param.firstChild.TEXT_NODE:
data = param.firstChild.data
config[param_name] = data
logging.debug("XML: '%s' -> '%s'" % (param_name, data))
if not config:
raise Exception, "Invalid configuration received from Splunk."
except Exception, e:
raise Exception, "Error getting Splunk configuration via STDIN: %s" % str(e)
return config
def run():
#The Main function that starts the action. The thread will sleep for however many seconds are configured via the Input.
# config = get_config()
#
#
# intervalone = config["intervalone"]
intervalone =60
while True:
weather_Connection(intervalone)
logging.info("Sleeping for %s seconds" %(intervalone))
time.sleep(float(intervalone))
if __name__ == '__main__':
if len(sys.argv) > 1:
if sys.argv[1] == "--scheme":
do_scheme()
else:
run()
sys.exit(0)

I've checked and tried your code and it works fine. Try replacing
logging.info("Sleeping for %s seconds" %(intervalone))
with
print("Sleeping for %s seconds" % (intervalone))
You should see this statement every 6 forecasts.
Note: why returning from weather_Connection() here
for i, (name, url) in enumerate(sorted(urls.iteritems())):
response = get_response(conn, url)
if not response:
print 'socket timeout on url#%d: %s' % (i, url)
return
responses[name] = response
You can just skip it with continue
for i, (name, url) in enumerate(sorted(urls.iteritems())):
response = get_response(conn, url)
if not response:
print 'socket timeout on url#%d: %s' % (i, url)
continue
responses[name] = response

IMAP search not finding new email

My test code sends an email with an attachment and saves a hash that is in the subject and body. I then have a function that takes the hash searches for it, gets the uid and fetches the email returning the attachment data.
The problem I am having is when I send a message and then subsequently search for the hash the email server says there is no matching uid, however if I run another copy of the script it does find it! Even if the second script is ran first! It first finds it but the original one doesn't; even though it is later!
Output
$ python test_server_file_functions.py
Creating mail server
S: '* OK Gimap ready for requests from [ip] [data]'
C: '0001 CAPABILITY'
S: '* CAPABILITY IMAP4rev1 UNSELECT IDLE NAMESPACE QUOTA ID XLIST CHILDREN X-GM-EXT-1 XYZZY SASL-IR AUTH=XOAUTH AUTH=XOAUTH2'
S: '0001 OK Thats all she wrote! [data]'
C: '0002 LOGIN "user#gmail.com" "password"'
S: '* CAPABILITY IMAP4rev1 UNSELECT IDLE NAMESPACE QUOTA ID XLIST CHILDREN X-GM-EXT-1 UIDPLUS COMPRESS=DEFLATE ENABLE MOVE'
S: '0002 OK user#gmail.com Anonymous Test authenticated (Success)'
C: '0003 SELECT INBOX'
S: '* FLAGS (\\Answered \\Flagged \\Draft \\Deleted \\Seen)'
S: '* OK [PERMANENTFLAGS (\\Answered \\Flagged \\Draft \\Deleted \\Seen \\*)] Flags permitted.'
S: '* OK [UIDVALIDITY 1] UIDs valid.'
S: '* 0 EXISTS'
S: '* 0 RECENT'
S: '* OK [UIDNEXT 132] Predicted next UID.'
S: '0003 OK [READ-WRITE] INBOX selected. (Success)'
Does not exists
Created mail server
Sending email
Sent email
Waiting 3 minutes to make sure it isn't a simple delay with the email being relayed
Downloading Data...
C: '0004 SEARCH SUBJECT "EMS Data ID: 622904923b1825d5742ed25fb792fafe2e710c40ceea09660a604be8fabac35ae9b006c43c7a992159b8b0df376383830a6d4c54ed5b141c8429a4feec89cd8b"'
S: '* SEARCH'
S: '0004 OK SEARCH completed (Success)'
Unhandled Error
Traceback (most recent call last):
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/mail/imap4.py", line 2455, in _defaultHandler
cmd.finish(rest, self._extraInfo)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/mail/imap4.py", line 382, in finish
d.callback((send, lastLine))
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/internet/defer.py", line 368, in callback
self._startRunCallbacks(result)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/internet/defer.py", line 464, in _startRunCallbacks
self._runCallbacks()
--- <exception caught here> ---
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/internet/defer.py", line 551, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/Users/user/Documents/gms/gms/mail_backend.py", line 178, in process_download_uid
raise IOError("Hash not found, however database indicates it was uploaded")
exceptions.IOError: Hash not found, however database indicates it was uploaded
There was an error retrieving the email
Traceback (most recent call last):
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/mail/imap4.py", line 2455, in _defaultHandler
cmd.finish(rest, self._extraInfo)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/mail/imap4.py", line 382, in finish
d.callback((send, lastLine))
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/internet/defer.py", line 368, in callback
self._startRunCallbacks(result)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/internet/defer.py", line 464, in _startRunCallbacks
self._runCallbacks()
--- <exception caught here> ---
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/internet/defer.py", line 551, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/Users/user/Documents/gms/gms/mail_backend.py", line 178, in process_download_uid
raise IOError("Hash not found, however database indicates it was uploaded")
exceptions.IOError: Hash not found, however database indicates it was uploaded
Quiting...
Code
import os
# logging
from twisted.python import log
import sys
import time
import email
from Utils import BLOCK
# IMAP
from IMAPBackend import connectToIMAPServer, Command
# SMTP
from SMTPBackend import connectToSMTPServer
# Hash Database
from HashDatabase import HashDatabase, hash
# deferreds
from twisted.internet.defer import Deferred, DeferredList, succeed
from twisted.internet.task import deferLater
#reactor
from twisted.internet import reactor
BLOCK_SIZE = BLOCK / 1024 # convert from bytes (needed for FTP) to kilobytes
def createMailServer(username, password, smtp_server, imap_server, hash_db = "hash.db"):
# create smtp connection
smtp_d = connectToSMTPServer(smtp_server, username, password)
# create imap connection
imap_d = connectToIMAPServer(imap_server, username, password)
dl = DeferredList([smtp_d, imap_d])
dl.addCallback(lambda r: [ MailServer(r[0][1], r[1][1], username, hash_db) ][0] )
return dl
class ServerManager(object):
def __init__(self, mail_servers):
self.mail_servers = mail_servers
def get_server(self, accnt):
for ms in self.mail_servers:
if ms.account == accnt:
return succeed(ms)
def return_server(self):
# retrieve the size avialable on the servers
get_space_deferreds = []
for ms in self.mail_servers:
d = ms.get_space()
d.addCallback(lambda r: (ms, r))
get_space_deferreds.append(d)
dl = DeferredList(get_space_deferreds, consumeErrors = True)
dl.addCallback(self.parse_sizes)
return dl
def parse_sizes(self, results):
for no_error, result in results:
server = result[0]
result = result[1]
if no_error:# not an error so a potential server
for argument in result[0]:
if argument[0] == "QUOTA":
print "Argument"
print argument
print "/Argument"
used, total = argument[2][1:3]
available_kb = int(total) - int(used)
if available_kb > BLOCK_SIZE:# server with more then our block size
return server
else:
print "Error from account %s" % server.email_address
# no free space was found :-(
raise IOError("No free space was found.")
class MailServer(object):
"Manages a server"
size = 0
used_space = 0
def __init__(self, smtp_connection, imap_connection, email_address, hash_db = "hash.db"):
self.smtp_connection = smtp_connection
self.imap_connection = imap_connection
self.hash_database = HashDatabase(hash_db)
self.email_address = email_address
self.account = email_address
# current uploads
self.current_uploads = {}
# current downloads
self.current_downloads = {}
def get_space(self):
cmd = Command("GETQUOTAROOT", "INBOX", ["QUOTAROOT", "QUOTA"])
d = self.imap_connection.sendCommand(cmd)
return d
def upload_data(self, data):
"""
Uploads data to email server returns deferred that will return with the imap uid
"""
data_hash = hash(data)
if data_hash in self.current_uploads:
d = Deferred()
self.current_uploads[data_hash].append(d)
return d
if self.hash_database.hash_in_list(data_hash):
print "Data hash is in the database; not uploading"
return succeed(data_hash)
else:
d = Deferred()
self.current_uploads[data_hash] = [d]
id = "EMS Data ID: %s" % data_hash
connection_deferred = self.smtp_connection.send_email(self.email_address, self.email_address, id, id, [["raw_ems", "ems.dat", data] ])
connection_deferred.addCallback(self.upload_success, data_hash)
connection_deferred.addErrback(self.upload_error, data_hash)
connection_deferred.addBoth(self.notify_uploaders, data_hash)
return d
def notify_uploaders(self, result, data_hash):
for waitingDeferred in self.current_uploads.pop(data_hash):
# if r is a Failure, this is equivalent to calling .errback with
# that Failure.
waitingDeferred.callback(result)
def upload_success(self, result, data_hash):
# add to hash table
self.hash_database.add_hash(data_hash)
# immediatly searching doesn't seem to work so search on data retrieval
return data_hash
def upload_error(self, error, data_hash):
# upload error
log.msg("Erroring uploading file")
log.err(error)
return error # send error to uploader
def download_data(self, data_hash):
"""
Downloads data from the email server returns a deferred that will return with the data
"""
d = Deferred()
if data_hash in self.current_downloads:
self.current_downloads[data_hash].append(d)
return d
if not self.hash_database.hash_in_list(data_hash):
print "Data Hash has never been uploaded..."
raise IOError("No such data hash exists")
else:
self.current_downloads[data_hash] = [d]
id = "EMS Data ID: %s" % data_hash
connection_deferred = self.imap_connection.search("SUBJECT", "\"EMS Data ID: %s\"" % data_hash, uid = False)
connection_deferred.addCallback(self.process_download_uid)
connection_deferred.addErrback(self.download_error, data_hash)
connection_deferred.addBoth(self.notify_downloaders, data_hash)
return d
return d
def process_download_uid(self, id):
if len(id) == 0:
raise IOError("Hash not found, however database indicates it was uploaded")
d = self.imap_connection.fetchMessage(id[-1])
d.addCallback(self.process_download_attachment, id[-1])
return d
def process_download_attachment(self, data, id):
email_text = data[id]["RFC822"]
msg = email.message_from_string(email_text)
for part in msg.walk():
type = part.get_content_type()
print repr(type)
if "raw_ems" in type:
log.msg("Found Payload")
return part.get_payload(decode = True)
log.msg("No attachment found")
raise IOError("Data not found")
def download_error(self, error, data_hash):
log.msg("Error downloading file")
log.err(error)
return error
def notify_downloaders(self, result, data_hash):
for waitingDeferred in self.current_downloads.pop(data_hash):
# if r is a Failure, this is equivalent to calling .errback with
# that Failure.
waitingDeferred.callback(result)
def delete_data(self, data_hash):
if not self.hash_database.hash_in_list(data_hash):
raise IOError("No such data hash uploaded")
else:
# delete it to prevent anyone from trying to download it while it is being deleted
self.hash_database.delete_hash(data_hash)
d = self.imap_connection.search("SUBJECT", "\"EMS Data ID: %s\"" % data_hash, uid = False)
d.addCallback(self.delete_message)
d.addErrback(self.deletion_error, data_hash)
return d
def deletion_error(self, error, data_hash):
print "Deletion Error"
log.err(error)
# restore hash to database
self.hash_database.add_hash(data_hash)
raise IOError("Couldn't delete message hash")
def delete_message(self, id):
if len(id) == 0:
raise IOError("Hash not found, however database indicates it was uploaded")
d = self.imap_connection.setFlags(id[-1], ["\\Deleted"])
d.addCallback(lambda result: self.imap_connection.expunge())
return d
## Main Code ##
if __name__ == "__main__":
def deleted_email(result):
print "Deleted the email succesfully"
print "====Result===="
print result
print "====Result===="
print "Quiting..."
os._exit(0)
def error_deleting(error):
print "There was an error deleting the email"
error.printTraceback()
print "Quiting..."
os._exit(0)
def retrieved_data(result, ms, hash):
print "Retrieved data"
print "=====Data===="
print result
print "Deleting email"
d = ms.delete_data(hash)
d.addCallback(deleted_email)
d.addErrback(error_deleting)
return d
def email_retrieval_error(error):
print "There was an error retrieving the email"
error.printTraceback()
print "Quiting..."
os._exit(0)
def sent_email(hash, ms):
print "Sent email"
print "Waiting 3 minutes to make sure it isn't a simple delay with the email being relayed"
time.sleep(3 * 60)
print "Downloading Data..."
d = ms.download_data(hash)
d.addCallback(retrieved_data, ms, hash)
d.addErrback(email_retrieval_error)
return d
def email_sending_error(error):
print "There was an error sending the email"
error.printTraceback()
print "Quiting..."
os._exit(0)
def mail_server_created(ms):
# created mail server
print "Created mail server"
print "Sending email"
d = ms.upload_data("this is the attachment data I am sending to my email account")
d.addCallback(sent_email, ms)
d.addErrback(email_sending_error)
return d
def mail_server_error(error):
print "Error creating mail server"
error.printTraceback()
print "Quiting..."
os._exit(0)
# create mail server object
print "Creating mail server"
d = createMailServer("user#gmail.com", "password", "smtp.gmail.com:587", "imap.gmail.com:993", hash_db = "testhash.db")
d.addCallback(mail_server_created)
d.addCallback(mail_server_error)
from twisted.internet import reactor
reactor.run()
I am thinking I may have to re-select the mailbox? I look in the RFC3501 select and search commands and found nothing about such a problem

Search command works on the data which is collected by parsing the entire mail folder which is been selected by Select command
You will have to select the mail folder again to get the mail entry updated.
Search result will not have the new mail input unless the server has implemented the IDLE/NOOP functionality (again it solely depends on mail server)