My test code sends an email with an attachment and saves a hash that is in the subject and body. I then have a function that takes the hash searches for it, gets the uid and fetches the email returning the attachment data.
The problem I am having is when I send a message and then subsequently search for the hash the email server says there is no matching uid, however if I run another copy of the script it does find it! Even if the second script is ran first! It first finds it but the original one doesn't; even though it is later!
Output
$ python test_server_file_functions.py
Creating mail server
S: '* OK Gimap ready for requests from [ip] [data]'
C: '0001 CAPABILITY'
S: '* CAPABILITY IMAP4rev1 UNSELECT IDLE NAMESPACE QUOTA ID XLIST CHILDREN X-GM-EXT-1 XYZZY SASL-IR AUTH=XOAUTH AUTH=XOAUTH2'
S: '0001 OK Thats all she wrote! [data]'
C: '0002 LOGIN "user#gmail.com" "password"'
S: '* CAPABILITY IMAP4rev1 UNSELECT IDLE NAMESPACE QUOTA ID XLIST CHILDREN X-GM-EXT-1 UIDPLUS COMPRESS=DEFLATE ENABLE MOVE'
S: '0002 OK user#gmail.com Anonymous Test authenticated (Success)'
C: '0003 SELECT INBOX'
S: '* FLAGS (\\Answered \\Flagged \\Draft \\Deleted \\Seen)'
S: '* OK [PERMANENTFLAGS (\\Answered \\Flagged \\Draft \\Deleted \\Seen \\*)] Flags permitted.'
S: '* OK [UIDVALIDITY 1] UIDs valid.'
S: '* 0 EXISTS'
S: '* 0 RECENT'
S: '* OK [UIDNEXT 132] Predicted next UID.'
S: '0003 OK [READ-WRITE] INBOX selected. (Success)'
Does not exists
Created mail server
Sending email
Sent email
Waiting 3 minutes to make sure it isn't a simple delay with the email being relayed
Downloading Data...
C: '0004 SEARCH SUBJECT "EMS Data ID: 622904923b1825d5742ed25fb792fafe2e710c40ceea09660a604be8fabac35ae9b006c43c7a992159b8b0df376383830a6d4c54ed5b141c8429a4feec89cd8b"'
S: '* SEARCH'
S: '0004 OK SEARCH completed (Success)'
Unhandled Error
Traceback (most recent call last):
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/mail/imap4.py", line 2455, in _defaultHandler
cmd.finish(rest, self._extraInfo)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/mail/imap4.py", line 382, in finish
d.callback((send, lastLine))
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/internet/defer.py", line 368, in callback
self._startRunCallbacks(result)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/internet/defer.py", line 464, in _startRunCallbacks
self._runCallbacks()
--- <exception caught here> ---
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/internet/defer.py", line 551, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/Users/user/Documents/gms/gms/mail_backend.py", line 178, in process_download_uid
raise IOError("Hash not found, however database indicates it was uploaded")
exceptions.IOError: Hash not found, however database indicates it was uploaded
There was an error retrieving the email
Traceback (most recent call last):
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/mail/imap4.py", line 2455, in _defaultHandler
cmd.finish(rest, self._extraInfo)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/mail/imap4.py", line 382, in finish
d.callback((send, lastLine))
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/internet/defer.py", line 368, in callback
self._startRunCallbacks(result)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/internet/defer.py", line 464, in _startRunCallbacks
self._runCallbacks()
--- <exception caught here> ---
File "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/twisted/internet/defer.py", line 551, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/Users/user/Documents/gms/gms/mail_backend.py", line 178, in process_download_uid
raise IOError("Hash not found, however database indicates it was uploaded")
exceptions.IOError: Hash not found, however database indicates it was uploaded
Quiting...
Code
import os
# logging
from twisted.python import log
import sys
import time
import email
from Utils import BLOCK
# IMAP
from IMAPBackend import connectToIMAPServer, Command
# SMTP
from SMTPBackend import connectToSMTPServer
# Hash Database
from HashDatabase import HashDatabase, hash
# deferreds
from twisted.internet.defer import Deferred, DeferredList, succeed
from twisted.internet.task import deferLater
#reactor
from twisted.internet import reactor
BLOCK_SIZE = BLOCK / 1024 # convert from bytes (needed for FTP) to kilobytes
def createMailServer(username, password, smtp_server, imap_server, hash_db = "hash.db"):
# create smtp connection
smtp_d = connectToSMTPServer(smtp_server, username, password)
# create imap connection
imap_d = connectToIMAPServer(imap_server, username, password)
dl = DeferredList([smtp_d, imap_d])
dl.addCallback(lambda r: [ MailServer(r[0][1], r[1][1], username, hash_db) ][0] )
return dl
class ServerManager(object):
def __init__(self, mail_servers):
self.mail_servers = mail_servers
def get_server(self, accnt):
for ms in self.mail_servers:
if ms.account == accnt:
return succeed(ms)
def return_server(self):
# retrieve the size avialable on the servers
get_space_deferreds = []
for ms in self.mail_servers:
d = ms.get_space()
d.addCallback(lambda r: (ms, r))
get_space_deferreds.append(d)
dl = DeferredList(get_space_deferreds, consumeErrors = True)
dl.addCallback(self.parse_sizes)
return dl
def parse_sizes(self, results):
for no_error, result in results:
server = result[0]
result = result[1]
if no_error:# not an error so a potential server
for argument in result[0]:
if argument[0] == "QUOTA":
print "Argument"
print argument
print "/Argument"
used, total = argument[2][1:3]
available_kb = int(total) - int(used)
if available_kb > BLOCK_SIZE:# server with more then our block size
return server
else:
print "Error from account %s" % server.email_address
# no free space was found :-(
raise IOError("No free space was found.")
class MailServer(object):
"Manages a server"
size = 0
used_space = 0
def __init__(self, smtp_connection, imap_connection, email_address, hash_db = "hash.db"):
self.smtp_connection = smtp_connection
self.imap_connection = imap_connection
self.hash_database = HashDatabase(hash_db)
self.email_address = email_address
self.account = email_address
# current uploads
self.current_uploads = {}
# current downloads
self.current_downloads = {}
def get_space(self):
cmd = Command("GETQUOTAROOT", "INBOX", ["QUOTAROOT", "QUOTA"])
d = self.imap_connection.sendCommand(cmd)
return d
def upload_data(self, data):
"""
Uploads data to email server returns deferred that will return with the imap uid
"""
data_hash = hash(data)
if data_hash in self.current_uploads:
d = Deferred()
self.current_uploads[data_hash].append(d)
return d
if self.hash_database.hash_in_list(data_hash):
print "Data hash is in the database; not uploading"
return succeed(data_hash)
else:
d = Deferred()
self.current_uploads[data_hash] = [d]
id = "EMS Data ID: %s" % data_hash
connection_deferred = self.smtp_connection.send_email(self.email_address, self.email_address, id, id, [["raw_ems", "ems.dat", data] ])
connection_deferred.addCallback(self.upload_success, data_hash)
connection_deferred.addErrback(self.upload_error, data_hash)
connection_deferred.addBoth(self.notify_uploaders, data_hash)
return d
def notify_uploaders(self, result, data_hash):
for waitingDeferred in self.current_uploads.pop(data_hash):
# if r is a Failure, this is equivalent to calling .errback with
# that Failure.
waitingDeferred.callback(result)
def upload_success(self, result, data_hash):
# add to hash table
self.hash_database.add_hash(data_hash)
# immediatly searching doesn't seem to work so search on data retrieval
return data_hash
def upload_error(self, error, data_hash):
# upload error
log.msg("Erroring uploading file")
log.err(error)
return error # send error to uploader
def download_data(self, data_hash):
"""
Downloads data from the email server returns a deferred that will return with the data
"""
d = Deferred()
if data_hash in self.current_downloads:
self.current_downloads[data_hash].append(d)
return d
if not self.hash_database.hash_in_list(data_hash):
print "Data Hash has never been uploaded..."
raise IOError("No such data hash exists")
else:
self.current_downloads[data_hash] = [d]
id = "EMS Data ID: %s" % data_hash
connection_deferred = self.imap_connection.search("SUBJECT", "\"EMS Data ID: %s\"" % data_hash, uid = False)
connection_deferred.addCallback(self.process_download_uid)
connection_deferred.addErrback(self.download_error, data_hash)
connection_deferred.addBoth(self.notify_downloaders, data_hash)
return d
return d
def process_download_uid(self, id):
if len(id) == 0:
raise IOError("Hash not found, however database indicates it was uploaded")
d = self.imap_connection.fetchMessage(id[-1])
d.addCallback(self.process_download_attachment, id[-1])
return d
def process_download_attachment(self, data, id):
email_text = data[id]["RFC822"]
msg = email.message_from_string(email_text)
for part in msg.walk():
type = part.get_content_type()
print repr(type)
if "raw_ems" in type:
log.msg("Found Payload")
return part.get_payload(decode = True)
log.msg("No attachment found")
raise IOError("Data not found")
def download_error(self, error, data_hash):
log.msg("Error downloading file")
log.err(error)
return error
def notify_downloaders(self, result, data_hash):
for waitingDeferred in self.current_downloads.pop(data_hash):
# if r is a Failure, this is equivalent to calling .errback with
# that Failure.
waitingDeferred.callback(result)
def delete_data(self, data_hash):
if not self.hash_database.hash_in_list(data_hash):
raise IOError("No such data hash uploaded")
else:
# delete it to prevent anyone from trying to download it while it is being deleted
self.hash_database.delete_hash(data_hash)
d = self.imap_connection.search("SUBJECT", "\"EMS Data ID: %s\"" % data_hash, uid = False)
d.addCallback(self.delete_message)
d.addErrback(self.deletion_error, data_hash)
return d
def deletion_error(self, error, data_hash):
print "Deletion Error"
log.err(error)
# restore hash to database
self.hash_database.add_hash(data_hash)
raise IOError("Couldn't delete message hash")
def delete_message(self, id):
if len(id) == 0:
raise IOError("Hash not found, however database indicates it was uploaded")
d = self.imap_connection.setFlags(id[-1], ["\\Deleted"])
d.addCallback(lambda result: self.imap_connection.expunge())
return d
## Main Code ##
if __name__ == "__main__":
def deleted_email(result):
print "Deleted the email succesfully"
print "====Result===="
print result
print "====Result===="
print "Quiting..."
os._exit(0)
def error_deleting(error):
print "There was an error deleting the email"
error.printTraceback()
print "Quiting..."
os._exit(0)
def retrieved_data(result, ms, hash):
print "Retrieved data"
print "=====Data===="
print result
print "Deleting email"
d = ms.delete_data(hash)
d.addCallback(deleted_email)
d.addErrback(error_deleting)
return d
def email_retrieval_error(error):
print "There was an error retrieving the email"
error.printTraceback()
print "Quiting..."
os._exit(0)
def sent_email(hash, ms):
print "Sent email"
print "Waiting 3 minutes to make sure it isn't a simple delay with the email being relayed"
time.sleep(3 * 60)
print "Downloading Data..."
d = ms.download_data(hash)
d.addCallback(retrieved_data, ms, hash)
d.addErrback(email_retrieval_error)
return d
def email_sending_error(error):
print "There was an error sending the email"
error.printTraceback()
print "Quiting..."
os._exit(0)
def mail_server_created(ms):
# created mail server
print "Created mail server"
print "Sending email"
d = ms.upload_data("this is the attachment data I am sending to my email account")
d.addCallback(sent_email, ms)
d.addErrback(email_sending_error)
return d
def mail_server_error(error):
print "Error creating mail server"
error.printTraceback()
print "Quiting..."
os._exit(0)
# create mail server object
print "Creating mail server"
d = createMailServer("user#gmail.com", "password", "smtp.gmail.com:587", "imap.gmail.com:993", hash_db = "testhash.db")
d.addCallback(mail_server_created)
d.addCallback(mail_server_error)
from twisted.internet import reactor
reactor.run()
I am thinking I may have to re-select the mailbox? I look in the RFC3501 select and search commands and found nothing about such a problem
Search command works on the data which is collected by parsing the entire mail folder which is been selected by Select command
You will have to select the mail folder again to get the mail entry updated.
Search result will not have the new mail input unless the server has implemented the IDLE/NOOP functionality (again it solely depends on mail server)
Related
I made a program to run a function every time a new email is received. The IMAP server, connection and inbox work as intended. it is also not an issue with the whole '"Inbox"' vs 'Inbox' format. I have tried almost everything I could think of with little to no success.
My code:
def connect():
global connected
#sets up connection with imap server
connection = imaplib.IMAP4_SSL("outlook.office365.com")
try:
(retcode, capabilities) = connection.login(login, secure_password)
except:
print(sys.exc_info()[1])
sys.exit(1)
print("connection successful")
connected = True
return connection
def disconnect(connection):
connection.logout()
connected = False
connection = connect()
while connected == True:
print("Searching for new Emails...")
connection.select(mailbox = "Inbox", readonly=False)
# test section start
result, data_for_update = connection.uid('search', None, "ALL")
ids = data_for_update[0]
id_list = ids.split()
if data_for_update[0].split()[-1] == latest_email_uid:
print('No new emails found. search again in 5 minutes')
time.sleep(300)
else:
result, data_for_update = connection.uid('fetch', latest_email_uid, '(RFC822)')
raw_email = data_for_update[0],[1]
latest_email_uid == data_for_update[0].split()[-1]
print('New email found')
time.sleep(300)
I tried to program to run the code above. But when I run it, it returns this error:
\lib\imaplib.py", line 1055, in _command_complete
raise self.error('%s command error: %s %s' % (name, typ, data))
imaplib.IMAP4.error: UID command error: BAD [b'Command Argument Error. 11']
iam trying to use a python script to do listener on gmail to get incoming emails with spicific criteria so far everything work well the script so listener and wait for the expected emai but when the email is recived i goten the following error any help will be welcome iam too beginer . already thanks.
the script used:
import time
from itertools import chain
import email
import imaplib
imap_ssl_host = 'imap.gmail.com' # imap.mail.yahoo.com
imap_ssl_port = 993
username = 'USERNAME or EMAIL ADDRESS'
password = 'PASSWORD'
# Restrict mail search. Be very specific.
# Machine should be very selective to receive messages.
criteria = {
'FROM': 'PRIVILEGED EMAIL ADDRESS',
'SUBJECT': 'SPECIAL SUBJECT LINE',
'BODY': 'SECRET SIGNATURE',
}
uid_max = 0
def search_string(uid_max, criteria):
c = list(map(lambda t: (t[0], '"'+str(t[1])+'"'), criteria.items())) + [('UID', '%d:*' % (uid_max+1))]
return '(%s)' % ' '.join(chain(*c))
# Produce search string in IMAP format:
# e.g. (FROM "me#gmail.com" SUBJECT "abcde" BODY "123456789" UID 9999:*)
def get_first_text_block(msg):
type = msg.get_content_maintype()
if type == 'multipart':
for part in msg.get_payload():
if part.get_content_maintype() == 'text':
return part.get_payload()
elif type == 'text':
return msg.get_payload()
server = imaplib.IMAP4_SSL(imap_ssl_host, imap_ssl_port)
server.login(username, password)
server.select('INBOX')
result, data = server.uid('search', None, search_string(uid_max, criteria))
uids = [int(s) for s in data[0].split()]
if uids:
uid_max = max(uids)
# Initialize `uid_max`. Any UID less than or equal to `uid_max` will be ignored subsequently.
server.logout()
# Keep checking messages ...
# I don't like using IDLE because Yahoo does not support it.
while 1:
# Have to login/logout each time because that's the only way to get fresh results.
server = imaplib.IMAP4_SSL(imap_ssl_host, imap_ssl_port)
server.login(username, password)
server.select('INBOX')
result, data = server.uid('search', None, search_string(uid_max, criteria))
uids = [int(s) for s in data[0].split()]
for uid in uids:
# Have to check again because Gmail sometimes does not obey UID criterion.
if uid > uid_max:
result, data = server.uid('fetch', uid, '(RFC822)') # fetch entire message
msg = email.message_from_string(data[0][1])
uid_max = uid
text = get_first_text_block(msg)
print 'New message :::::::::::::::::::::'
print text
server.logout()
time.sleep(1)
the result:
C:\Users\PC Sony>"C:/Users/PC Sony/AppData/Local/Programs/Python/Python38/python.exe" "c:/Users/PC Sony/Desktop/elzero/elzero/# import PySimpleGUI.py"
Traceback (most recent call last):
File "c:/Users/PC Sony/Desktop/elzero/elzero/# import PySimpleGUI.py", line 68, in <module>
result, data = server.uid('fetch', uid, '(RFC822)') # fetch entire message
File "C:\Users\PC Sony\AppData\Local\Programs\Python\Python38\lib\imaplib.py", line 881, in uid
typ, dat = self._simple_command(name, command, *args)
File "C:\Users\PC Sony\AppData\Local\Programs\Python\Python38\lib\imaplib.py", line 1205, in _simple_command
return self._command_complete(name, self._command(name, *args))
File "C:\Users\PC Sony\AppData\Local\Programs\Python\Python38\lib\imaplib.py", line 963, in _command
data = data + b' ' + arg
TypeError: can't concat int to bytes
result, data = server.uid('fetch', uid, '(RFC822)')
should be changed to
result, data = server.uid('fetch', str(uid), '(RFC822)')
arguments should be of type string or bytes.
source code in imaplib.py
for arg in args:
if arg is None: continue
if isinstance(arg, str):
arg = bytes(arg, "ASCII")
data = data + b' ' + arg
I have a script which is supposed to login via imap, and scan my mailbox for email addresses i have sent mail to. It seems there is an error in the code below. Would it be best to just scan named boxes or is there another fix?
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# imap-email-address-collector
#
import sys
import re
import csv
import getpass
import imaplib
import argparse
from email.parser import HeaderParser
RE_EMAIL = re.compile(r'^[a-z0-9._%+-]+\#[a-z0-9.-]+\.[a-z]{2,}$')
RE_QUOTES = re.compile(r'[\'\"]')
RE_SPACES = re.compile(r'[\n\t\s]+')
NOSSL_PROMT = 'LETTHEWOLDREADALLMESSAGES'
results = {}
unmatched = set()
def matchAndAdd(email, name=''):
email = email.lower()
if RE_EMAIL.match(email):
if email not in results or len(name) > len(results[email]): # only overwrite with longer name
results[email] = name.strip()
else:
unmatched.add(email)
def grabAddress(address):
address = str(address).strip()
address = RE_QUOTES.sub('', address)
address = RE_SPACES.sub(' ', address)
if address.startswith('<'): # No name, just an email address
address = address[1:]
if address.endswith('>'):
address = address[:-1]
matchAndAdd(address)
else: # Name and email address
try:
name, email = address.split('<')
except Exception:
matchAndAdd(address)
else:
matchAndAdd(email[:-1], name)
def listBoxes(imap):
boxes = imap.list()
if boxes[0] == 'OK' and len(boxes) > 1:
for box in boxes[1]:
box = re.split(r'\) \".\" ', box, maxsplit=1)
if len(box) == 2:
yield box[1]
else:
print 'No folders found.'
sys.exit(0)
def main(args):
if args.password:
password = args.password
else:
password = getpass.getpass('Password: ')
if args.nossl:
print 'Connecting to %s:%s without SSL...' % (args.host, args.port)
if not args.donotannoyme:
confirmation = raw_input('Please type %s:' % NOSSL_PROMT)
if not confirmation == NOSSL_PROMT:
print 'Good choice ;) (Disable with --donotannoyme)'
sys.exit(1)
imap = imaplib.IMAP4(args.host, args.port)
else:
print 'Connecting to %s:%s over SSL...' % (args.host, args.port)
imap = imaplib.IMAP4_SSL(args.host, args.port)
try:
imap.login(args.user, password)
except imaplib.IMAP4.error:
print "Login failed."
sys.exit(1)
else:
print 'Logged in as %s' % args.user
print 'Collecting email addresses from all messages...'
for box in listBoxes(imap):
print 'Scanning %s' % box
imap.select(box, readonly=True)
typ, data = imap.search(None, 'Inbox')
count = 0
for num in data[0].split():
typ, data = imap.fetch(num, '(BODY[HEADER.FIELDS (TO FROM)])')
headers = headerParser.parsestr(data[0][1])
for h in ('From', 'To'):
if headers[h]:
for address in headers[h].split(','):
grabAddress(address)
count += 1
sys.stdout.flush()
sys.stdout.write('\rScanned %s messages' % count)
if count > 0:
print ''
imap.close()
imap.logout()
if len(results) > 0:
print 'Found %s addresses' % len(results)
if not args.csv:
toStdout = True
else:
try:
outFile = open(args.csv, 'wb')
except Exception:
toStdout = True
print 'Cannot write to %s, dumping out here.' % args.csv
else:
toStdout = False
if toStdout:
outFile = sys.stdout
print '======================================================='
else:
print 'Writing to %s' % args.csv
writer = csv.writer(outFile)
writer.writerows(list(results.items()))
if toStdout:
print '======================================================='
else:
outFile.close()
if len(unmatched) > 0:
print 'Could not interpret %s address(es): %s' % (len(unmatched), "'"+("', '".join(unmatched))+"'")
else:
print 'No addresses found'
if __name__ == '__main__':
headerParser = HeaderParser()
argParser = argparse.ArgumentParser()
argParser.add_argument('--host', help='imap host address', required=True)
argParser.add_argument('--user', help='login username', required=True)
argParser.add_argument('--csv', help='(optional) output csv filepath')
argParser.add_argument('--nossl', help='(optional) do not use ssl', action='store_true')
argParser.add_argument('--donotannoyme', help='(optional) do not complain about non-ssl connections', action='store_true')
argParser.add_argument('--password', help='(optional) login password (will be prompted otherwise)')
argParser.add_argument('--port', help='(optional) imap host port, defaults to 993', type=int, default=993)
main(argParser.parse_args())
However, when i execute the script i get the error below. What may be the solution to this?
Traceback (most recent call last):
File "./imap.py", line 169, in <module>
main(argParser.parse_args())
File "./imap.py", line 99, in main
typ, data = imap.search(None, 'Inbox')
File "/usr/lib64/python2.7/imaplib.py", line 640, in search
typ, dat = self._simple_command(name, *criteria)
File "/usr/lib64/python2.7/imaplib.py", line 1083, in _simple_command
return self._command_complete(name, self._command(name, *args))
File "/usr/lib64/python2.7/imaplib.py", line 918, in _command_complete
raise self.error('%s command error: %s %s' % (name, typ, data))
imaplib.error: SEARCH command error: BAD ['Error in IMAP command SEARCH: Unknown argument INBOX (0.001 + 0.000 secs).']
I am using Quickfix and I modified my toAdmin function to insert the username and password into the logon message. I adapted my code from the c++ instructions but I got a weird getHeader() attribute error.
The traceback is the following :
<20151223-10:48:31.142, FIX.4.2:MATHCLIENT1->CSTEST, event>
(Created session)
Type 1 for order , 2 to exit and d to debug.
<20151223-10:48:31.149, FIX.4.2:CLIENT1->TEST, event>
(Connecting to hostX on port Y)
Traceback (most recent call last):
File "initiator.py", line 28, in toAdmin
message.getHeader ().getField (msgType)
File "C:\Users\user\Anaconda\lib\site-packages\quickfix.py", line 27015, in <lambda>
__getattr__ = lambda self, name: _swig_getattr(self, SessionID, name)
File "C:\Users\user\Anaconda\lib\site-packages\quickfix.py", line 57, in _swig_getattr
raise AttributeError(name)
AttributeError: getHeader
My code follows below :
import sys
import time
import thread
import argparse
from datetime import datetime
import quickfix as fix
class Application(fix.Application):
orderID = 0
execID = 0
def gen_ord_id(self):
global orderID
orderID+=1
return orderID
def onCreate(self, sessionID):
return
def onLogon(self, sessionID):
self.sessionID = sessionID
print ("Successful Logon to session '%s'." % sessionID.toString())
return
def onLogout(self, sessionID): return
def toAdmin(self, sessionID, message):
msgType = fix.MsgType ()
message.getHeader ().getField (msgType)
if (msgType.getValue () == fix.MsgType_Logon):
print 'Logging on.'
# message.setField (fix.Password (settings.get (self.sessionID).getString ("Password")))
# message.setField (fix.Username (settings.get (self.sessionID).getString ("Username")))
message.setField(fix.Password('password'))
message.setField(fix.Username('username'))
message.setField (fix.ResetSeqNumFlag (True))
return
def fromAdmin(self, sessionID, message):
return
def toApp(self, sessionID, message):
print "Sent the following message: %s" % message.toString()
return
def fromApp(self, message, sessionID):
print "Received the following message: %s" % message.toString()
return
def genOrderID(self):
self.orderID = self.orderID+1
return `self.orderID`
def genExecID(self):
self.execID = self.execID+1
return `self.execID`
def put_order(self):
print("Creating the following order: ")
trade = fix.Message()
trade.getHeader().setField(fix.BeginString(fix.BeginString_FIX50)) #
trade.getHeader().setField(fix.MsgType(fix.MsgType_NewOrderSingle)) #39=D
trade.setField(fix.ClOrdID(self.genExecID())) #11=Unique order
trade.setField(fix.HandlInst(fix.HandlInst_MANUAL_ORDER_BEST_EXECUTION)) #21=3 (Manual order, best executiona)
trade.setField(fix.Symbol('SMBL')) #55=SMBL ?
trade.setField(fix.Side(fix.Side_BUY)) #43=1 Buy
trade.setField(fix.OrdType(fix.OrdType_LIMIT)) #40=2 Limit order
trade.setField(fix.OrderQty(100)) #38=100
trade.setField(fix.Price(10))
print trade.toString()
fix.Session.sendToTarget(trade, self.sessionID)
def main(config_file):
try:
settings = fix.SessionSettings(config_file)
application = Application()
storeFactory = fix.FileStoreFactory(settings)
# logFactory = fix.FileLogFactory(settings)
logFactory = fix.ScreenLogFactory(settings)
initiator = fix.SocketInitiator(application, storeFactory, settings, logFactory)
initiator.start()
print 'Type 1 for order , 2 to exit and d to debug.'
while 1:
input = raw_input()
if input == '1':
print "Putin Order"
application.put_order()
if input == '2':
sys.exit(0)
if input == 'd':
import pdb
pdb.set_trace()
else:
print "Valid input is 1 for order, 2 for exit"
continue
except (fix.ConfigError, fix.RuntimeError), e:
print e
if __name__=='__main__':
# logfile = open('errorlog.txt','w')
# original_stderr = sys.stderr
# sys.stderr = logfile
parser = argparse.ArgumentParser(description='FIX Client')
parser.add_argument('file_name', type=str, help='Name of configuration file')
args = parser.parse_args()
main(args.file_name)
# sys.stderr = original_stderr
# logfile.close()
This looks mostly correct, however, I believe you have entered your arguments in the wrong order. Try defining your function as follows:
def toAdmin(self, message, sessionID):
I'm currently in the process of learning ssh via the brute-force/ just keep hacking until I understand it approach. After some trial and error I've been able to successfully send a "pty-req" followed by a "shell" request, I can get the login preamble, send commands and receive stdout but I'm not exactly sure how to tell the SSH service I want to recieve stderr and status messages. Reading through other SSH implementations ( paramiko, Net::SSH ) hasn't been much of a guide at the moment.
That said, looking at one of the RFC's for SSH, I believe that perhaps one of the listed requests might be what I am looking for: https://www.rfc-editor.org/rfc/rfc4250#section-4.9.3
#!/usr/bin/env python
from twisted.conch.ssh import transport
from twisted.conch.ssh import userauth
from twisted.conch.ssh import connection
from twisted.conch.ssh import common
from twisted.conch.ssh.common import NS
from twisted.conch.ssh import keys
from twisted.conch.ssh import channel
from twisted.conch.ssh import session
from twisted.internet import defer
from twisted.internet import defer, protocol, reactor
from twisted.python import log
import struct, sys, getpass, os
log.startLogging(sys.stdout)
USER = 'dward'
HOST = '192.168.0.19' # pristine.local
PASSWD = "password"
PRIVATE_KEY = "~/id_rsa"
class SimpleTransport(transport.SSHClientTransport):
def verifyHostKey(self, hostKey, fingerprint):
print 'host key fingerprint: %s' % fingerprint
return defer.succeed(1)
def connectionSecure(self):
self.requestService(
SimpleUserAuth(USER,
SimpleConnection()))
class SimpleUserAuth(userauth.SSHUserAuthClient):
def getPassword(self):
return defer.succeed(PASSWD)
def getGenericAnswers(self, name, instruction, questions):
print name
print instruction
answers = []
for prompt, echo in questions:
if echo:
answer = raw_input(prompt)
else:
answer = getpass.getpass(prompt)
answers.append(answer)
return defer.succeed(answers)
def getPublicKey(self):
path = os.path.expanduser(PRIVATE_KEY)
# this works with rsa too
# just change the name here and in getPrivateKey
if not os.path.exists(path) or self.lastPublicKey:
# the file doesn't exist, or we've tried a public key
return
return keys.Key.fromFile(filename=path+'.pub').blob()
def getPrivateKey(self):
path = os.path.expanduser(PRIVATE_KEY)
return defer.succeed(keys.Key.fromFile(path).keyObject)
class SimpleConnection(connection.SSHConnection):
def serviceStarted(self):
self.openChannel(SmartChannel(2**16, 2**15, self))
class SmartChannel(channel.SSHChannel):
name = "session"
def getResponse(self, timeout = 10):
self.onData = defer.Deferred()
self.timeout = reactor.callLater( timeout, self.onData.errback, Exception("Timeout") )
return self.onData
def openFailed(self, reason):
print "Failed", reason
#defer.inlineCallbacks
def channelOpen(self, ignoredData):
self.data = ''
self.oldData = ''
self.onData = None
self.timeout = None
term = os.environ.get('TERM', 'xterm')
#winsz = fcntl.ioctl(fd, tty.TIOCGWINSZ, '12345678')
winSize = (25,80,0,0) #struct.unpack('4H', winsz)
ptyReqData = session.packRequest_pty_req(term, winSize, '')
try:
result = yield self.conn.sendRequest(self, 'pty-req', ptyReqData, wantReply = 1 )
except Exception as e:
print "Failed with ", e
try:
result = yield self.conn.sendRequest(self, "shell", '', wantReply = 1)
except Exception as e:
print "Failed shell with ", e
#fetch preample
data = yield self.getResponse()
"""
Welcome to Ubuntu 11.04 (GNU/Linux 2.6.38-8-server x86_64)
* Documentation: http://www.ubuntu.com/server/doc
System information as of Sat Oct 29 13:09:50 MDT 2011
System load: 0.0 Processes: 111
Usage of /: 48.0% of 6.62GB Users logged in: 1
Memory usage: 39% IP address for eth1: 192.168.0.19
Swap usage: 3%
Graph this data and manage this system at https://landscape.canonical.com/
New release 'oneiric' available.
Run 'do-release-upgrade' to upgrade to it.
Last login: Sat Oct 29 01:23:16 2011 from 192.168.0.17
"""
print data
while data != "" and data.strip().endswith("~$") == False:
try:
data = yield self.getResponse()
print repr(data)
"""
\x1B]0;dward#pristine: ~\x07dward#pristine:~$
"""
except Exception as e:
print e
break
self.write("false\n")
#fetch response
try:
data = yield self.getResponse()
except Exception as e:
print "Failed to catch response?", e
else:
print data
"""
false
\x1B]0;dward#pristine: ~\x07dward#pristine:~$
"""
self.write("true\n")
#fetch response
try:
data = yield self.getResponse()
except Exception as e:
print "Failed to catch response?", e
else:
print data
"""
true
\x1B]0;dward#pristine: ~\x07dward#pristine:~$
"""
self.write("echo Hello World\n\x00")
try:
data = yield self.getResponse()
except Exception as e:
print "Failed to catch response?", e
else:
print data
"""
echo Hello World
Hello World
\x1B]0;dward#pristine: ~\x07dward#pristine:~$
"""
#Close up shop
self.loseConnection()
dbgp = 1
def request_exit_status(self, data):
status = struct.unpack('>L', data)[0]
print 'status was: %s' % status
def dataReceived(self, data):
self.data += data
if self.onData is not None:
if self.timeout and self.timeout.active():
self.timeout.cancel()
if self.onData.called == False:
self.onData.callback(data)
def extReceived(self, dataType, data):
dbgp = 1
print "Extended Data recieved! dataType = %s , data = %s " % ( dataType, data, )
self.extendData = data
def closed(self):
print 'got data : %s' % self.data.replace("\\r\\n","\r\n")
self.loseConnection()
reactor.stop()
protocol.ClientCreator(reactor, SimpleTransport).connectTCP(HOST, 22)
reactor.run()
Additionally I tried adding in an explicit bad command to the remote shell:
self.write("ls -alF badPathHere\n\x00")
try:
data = yield self.getResponse()
except Exception as e:
print "Failed to catch response?", e
else:
print data
"""
ls -alF badPathHere
ls: cannot access badPathHere: No such file or directory
\x1B]0;dward#pristine: ~\x07dward#pristine:~$
"""
And it looks like stderr is being mixed into stderr
Digging through the source code for OpenSSH, channel session logic is handled in session.c at line
2227 function -> session_input_channel_req which if given a pty-req then a "shell" request leads to do_exec_pty which ultimately leads to the call to session_set_fds(s, ptyfd, fdout, -1, 1, 1). The forth argument would normally be a file descriptor responsible for handling stderr but since none is supplied then there won't be any extended data for stderr.
Ultimately, even if I modified openssh to provide a stderr FD, the problem resides with the shell. Complete guess work at this point but I believe that similar to logging into a ssh service via a terminal like xterm or putty, that stderr and stdout are sent together unless explicitly redirected via something like "2> someFile" which is beyond the scope of a SSH service provider.