(Python 3)
I am using a Python generator to read messages from a queue.
After the consumer reads a queue message, it needs to be able to tell the generator to delete the queue message if it was successfully processed.
In order to .send() to a Python generator, it seems I must first .send(None) to the generator. This is making my code fatter than I think it should be.
Can anyone suggest a way for qconsumer.py to drive the generator with fewer lines of code? I have identified which lines below I am hoping to eliminate.
In short, how can I make the code below more compact, any suggestions for how I can delete lines?
Code below is qconsumer.py:
from qserver import Qserver
myqserver = Qserver()
myproducer = myqserver.producer() # trying to eliminate this line
# first send to a generator must be None
myproducer.send(None) # trying to eliminate this line
for msg in myproducer:
# do something with message
print(msg)
if messageprocessok:
myproducer.send('delete')
Code below is qserver.py:
# -*- coding: utf-8 -*-
import boto
from boto.sqs.connection import SQSConnection
from boto.sqs.message import Message
QNAME = 'qinbound'
SQSREGION = 'us-west-1'
class Qserver():
"""A simple Q server."""
def __init__(self, qname=None, sqsregion=None):
self.qname = qname or QNAME
self.sqsregion = sqsregion or SQSREGION
self.sqsconn = boto.sqs.connect_to_region(self.sqsregion)
self.q_in = self.sqsconn.get_queue(self.qname)
def producer(self):
while True:
qmessage = self.q_in.read(wait_time_seconds=20)
if qmessage is None:
continue
action = (yield qmessage.get_body())
if action == 'delete':
# if processing completed ok, clear message from this queue
self.q_in.delete_message(qmessage)
Your current consumer is throwing away messages because each send call returns one. You should do this instead:
myqserver = Qserver()
myproducer = myqserver.producer()
messageprocessok = False
while True:
msg = myproducer.send('delete' if messageprocessok else None)
# do something with message
print(msg)
or alternatively:
myqserver = Qserver()
myproducer = myqserver.producer()
msg = next(myproducer)
while True:
# do something with message
print(msg)
msg = myproducer.send('delete' if messageprocessok else None)
The fact that you need separate calls to Qserver() and myqserver.producer() is simply because you made prouducer a method of a class. Alternatively you could use a stand-alone function, or make a wrapper function that simply returns Qserver().producer(). Here's the stand-alone version:
def producer(qname=None, sqsregion=None):
qname = qname or QNAME
sqsregion = sqsregion or SQSREGION
sqsconn = boto.sqs.connect_to_region(sqsregion)
q_in = sqsconn.get_queue(qname)
while True:
qmessage = q_in.read(wait_time_seconds=20)
if qmessage is None:
continue
action = (yield qmessage.get_body())
if action == 'delete':
# if processing completed ok, clear message from this queue
q_in.delete_message(qmessage)
Having understood what you're trying to do, I think I would avoid mixing send with iteration. Having the myqserver class be an iterator itself seems to make more sense to me:
# -*- coding: utf-8 -*-
import boto
from boto.sqs.connection import SQSConnection
from boto.sqs.message import Message
QNAME = 'qinbound'
SQSREGION = 'us-west-1'
class Qserver():
"""A simple Q server."""
_current_message = None
def __init__(self, qname=None, sqsregion=None):
self.qname = qname or QNAME
self.sqsregion = sqsregion or SQSREGION
self.sqsconn = boto.sqs.connect_to_region(self.sqsregion)
self.q_in = self.sqsconn.get_queue(self.qname)
def __iter__(self):
return self
def __next__(self):
while True:
qmessage = self.q_in.read(wait_time_seconds=20)
if qmessage is not None:
self._current_message = qmessage
return qmessage
next = __next__
def delete_current(self):
if self._current_message is not None:
self.q_in.delete_message(self._current_message)
And usage will be something like:
from qserver import Qserver
myqserver = Qserver()
for msg in myqserver:
# do something with message
print(msg)
if messageprocessok:
myqserver.delete_current()
Related
I am trying to use the value of a variable, which is a string, as an argument and I keep getting "'Str' not callable' error. I haven't used str as a variable name I can make the code work with eval, however, I've read dire warnings about eval, so am unsure what to do. My code is below.
from time import sleep
from binance.client import Client
from binance.websockets import BinanceSocketManager
class s33():
def __init__(self):
self.client = Client("", "")
self.bm = BinanceSocketManager(self.client)
def process_trade_message(self, message):
global count, conn_key
print(count)
if count >= 10:
print('closing socket')
# use either stop_socket or close, or both
self.bm.stop_socket(conn_key)
self.bm.close()
# reset the count
count = 0
def go(self, sockb):
global count, conn_key
print(sockb['1'])
sock = 'self.bm.'+sockb['1']
print(sock)
count = 0
conn_key = sock(self.process_trade_message)
self.bm.start()
if __name__ == '__main__':
while True:
s = s33()
socka = {'1':'start_miniticker_socket'}
s.go(socka)
sleep(20)
print('sleeping')
I have read people recommend using a dict, So I passed the dict as the arg and tried to extract the string in the function,which is the code below. I tried to extract the string and pass that as an arg to the function. s.go(socka['1'], I tried passing the just the variable as an arg, socka = 'start_miniticker_socket' and I can get that to work if I use eval('self.bm'+socka) I tried the percentage sign with no luck. Not sure how to do this without using eval. I am still fairly new and can't find an alternative answer after several hours of searching that works. any help would be appreciated.
I think what people meant when suggesting a dict is something like this:
class s33():
# Codes...
def go(self, func_name):
global count, conn_key
print(func_name)
mapper = {
'start_miniticker_socket': self.bm.start_miniticker_socket
}
# Get self.bm.start_miniticker or None
sock = mapper.get(func_name)
print(sock)
if sock:
count = 0
conn_key = sock(self.process_trade_message)
self.bm.start()
else:
pass # Handle when sock is None
if __name__ == '__main__':
while True:
s = s33()
socka = 'start_miniticker_socket'
s.go(socka)
sleep(20)
print('sleeping')
I'm trying to pass a string in Python that was obtained from netstat to awk for use in building a new object.
Any clue why this isn't working? Please excuse the horrible coding, I just started using Python today and trying to learn how to use the language.
class NetstatGenerator():
def __init__(self):
import subprocess
self.results = subprocess.Popen("netstat -anbp tcp", shell=True, stdout=subprocess.PIPE).stdout.read()
self.list = []
self.parse_results()
def parse_results(self):
lines = self.results.splitlines(True)
for line in lines:
if not str(line).startswith("tcp"):
print("Skipping")
continue
line_data = line.split(" ")
self.list.append(NetstatData(self.line_data[0], self.line_data[15], self.line_data[18], self.line_data[23]))
def get_results(self):
return self.list
class NetstatData():
def __init__(self, protocol, local_address, foreign_address, state):
self.protocol = protocol
self.local_address = local_address
self.foreign_address = foreign_address
self.state = state
def get_protocol(self):
return str(self.protocol)
def get_local_address(self):
return str(self.local_address)
def get_foreign_address(self):
return str(self.foreign_address)
def get_state(self):
return str(self.state)
data = NetstatGenerator()
Sorry, netstat does not support -b on Linux, and I don't have a BSD box lying around.
Let's assume you have a list of lines, called netstat_output, with items like this:
tcp 0 0 127.0.0.1:9557 127.0.0.1:56252 ESTABLISHED -
To parse a single line, you split() it and pick elements at indexes 0, 3, 4, 5.
To store the items, you don't need to define a boilerplate holding class; namedtuple does what you want:
from collections import namedtuple
NetstatInfo = namedtuple('NetstatInfo',
['protocol', 'local_address', 'remote_address', 'state'])
Now you can parse a line:
def parseLine(line):
fields = line.split()
if len(fields) == 7 and fields[0] in ('tcp', 'udp'):
# alter this condition to taste;
# remember that netstat injects column headers.
# consider other checks, too.
return NetstatInfo(fields[0], fields[3], fields[4], fields[5])
# otherwise, this function implicitly returns None
Now something like this must be possible:
result = []
for line in subprocess.Popen(...):
item = parseLine(line)
if line: # parsed successfully
result.append(line)
# now result is what you wanted; e.g. access result[0].remote_address
When I execute this program I get an empty list:
I am expecting it to create the list of objects and append the objects to the obj_list_addresses list.
Then when I call the get_good_addresses() I expect it to go back through that list and execute code on each object in the list only the list returns empty [] almost like its getting overwritten.
I am fairly new to python and know that I am missing something important.
Main:
from address import Address
from address_processor import AddressProcessor
addresses=[]
addresses = open('test.txt').read().splitlines()
proccess_addresses = AddressProcessor(addresses)
proccess_addresses.create_addresses_obj()
proccess_addresses.get_good_addresses()
AddressProcessor Class:
import multiprocessing
from address import Address
class AddressProcessor(object):
"""AddressProcessor will process a txt file with addresses"""
def __init__(self, addresses):
self.addresses = addresses
self.return_addresses = []
self.obj_list_addresses = []
def create_addresses_obj(self):
jobs = []
for address in self.addresses:
process = multiprocessing.Process(target=self.worker, args=(address,))
jobs.append(process)
process.start()
for job in jobs:
job.join()
print('created objects for addresses in text file')
def worker(self, address):
obj = Address(address)
self.obj_list_addresses.append(obj)
def get_good_addresses(self):
print self.obj_list_addresses
Address Class:
from string import replace
from pprint import pprint
class Address(object):
"""
This is address class send it an address it will look up
the addy and return json string of the parcels that matched the address
then update status if it was the only one returned its good if not its bad
"""
def __init__(self, address):
self.address = address
self.status = ''
self.json_string = ''
self.set_json_string()
def get_address(self):
return self.address
def set_json_string(self):
r = requests.get('urlbasegoeshere'+replace(self.address," ","+")+'&pagesize=40&page=1')
self.json_string = r.json
self.set_status()
def set_status(self):
if len(self.json_string) == 1:
self.status = 1
elif len(self.json_string)!=1:
self.status = 0
def get_status(self):
return self.status
Why are you using 'multiprocessing' to create address objects? Different process don't share memory, i.e. they don't share objects. This is not a python thing, it's the same whatever language you use.
Replace these three lines
process = multiprocessing.Process(target=self.worker, args=(address,))
jobs.append(process)
process.start()
with
self.worker(address)
I've been trying to optimize my application, and although I made the function run on average 10.06 seconds when I profiled it by itself, when it is put on a QThread, it takes around 17-22 seconds.
It's 2x slower in the QThread. How do I fix that?
The function is actually initializing a class called DocxDocument, which is a document that I read from a Word file and parsed it for my needs.
I have a QThread that creates this class and uses Qt signals to send progress information back to the GUI. Here is the code from that class:
class DocxImporterThread(QThread):
'''
This thread is used to import a .docx document, report its progress, and
then return the document that it parsed.
'''
reportProgress = pyqtSignal(int)
reportError = pyqtSignal(Exception, str)
reportText = pyqtSignal(str)
def __init__(self, filePath):
QThread.__init__(self)
self.setPriority(QThread.HighestPriority)
self._filePath = filePath
self._docx = None
self._html = ''
self._bookmarks = None
self._pages = None
self._stop = False
def run(self):
def myProgressHook(percentage):
self.reportProgress.emit(percentage)
def myCancelHook():
return self._stop
try:
self._docx = DocxDocument(self._filePath, myProgressHook, myCancelHook)
if not self._stop:
self._html = self._docx.getMainPage()
self._bookmarks = self._docx.getHeadings()
self._pages = self._docx.getPages()
except Exception as ex2:
print 'ERROR: The .docx document didn\'t import.'
self.reportError.emit(ex2, traceback.format_exc())
The getMainPage(), getHeadings(), and getPages() are instantaneous because they just return a reference to something that the constructor already created. Here is the code I used to profile my DocxDocument class:
testFile = 'some_file.docx'
statSave = 'profile.out'
def progress(percent):
print ' --', percent, '--'
cProfile.run('DocxDocument(testFile)', filename=statSave)
myStats = pstats.Stats(statSave)
myStats.sort_stats('cumulative', 'name')
myStats.print_stats()
Thanks for your time in looking at this!
I am creating a python script that grabs information from an API and creates a context menu that gives you access to them. I want to use threading as it runs a little slow on the one call to the API, but I am not sure how to implement threading with my code. I am using this site for threading reference: http://www.ibm.com/developerworks/aix/library/au-threadingpython/ I understand the logic in the code I just don't want to write a threading class for every method that I want threaded.
Here is the class that creates the context menu and then parses the json returned, I think I should add it to the for loop in the run command. Any help is greatly appreciated.
class SyncsnippetsCommand(sublime_plugin.TextCommand):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
def buildLexerDict(self,snippets):
lexers = snippets[0]['user']['lexers']
lexer_dict = {}
for lexer in lexers:
lexer_dict[lexer] = []
return lexer_dict
def buildsnippetsContextDict(self,snippets,lexer_dict):
snippets_dict = lexer_dict
for snippet in snippets:
snippets_dict[snippet['lexer']].append({"id":str(snippet['id']),
"title":snippet['title']})
return snippets_dict
def run(self, edit):
snippet_url = buildsnippetURL()
snippets_count = 1;
snippets = getsnippets(snippet_url)
context_menu = '['
context_menu += '\n\t{ "caption": "snippets", "id": "file", "children":'
context_menu += '\n\t\t['
if snippets == None:
{"caption":"No snippets available"}
else:
snippets = snippets['objects']
lexers = self.buildLexerDict(snippets)
snippets_dict = self.buildsnippetsContextDict(snippets, lexers)
for j,key in reversed(list(enumerate(reversed(snippets_dict.keys())))):
... loop through JSON and create menu ...
if j == 0:
context_menu += ''
else:
context_menu += ','
context_menu += '\n\t\t]'
context_menu += '\n\t}'
context_menu += '\n]'
f = open(sublime.packages_path() + '\snippetSync\\Context.sublime-menu', 'w')
f.write(context_menu)
f.close
self.view.set_status('snippet', 'snippet Sync: Added ' + str(snippets_count) + ' snippets from your account.')
sublime.set_timeout(lambda: self.view.erase_status('snippet'), 3000)
return
Here is a simple Sublime Text 2 plugin with threading. What it does is insert Hello World! after 3 seconds. What you'll notice is that you can still move the cursor during those three seconds.
In your case, it looks like you just need to grab a bunch of snippets from an API and create a context menu from the returned data. Then there will be a notification at the bottom telling you how many snippets were added. I could be wrong, but you should be able to modify this code to make your plugin work.
import threading
import time
import sublime
import sublime_plugin
"""
The command just creates and runs a thread.
The thread will do all the work in the background.
Note that in your Thread constructor, you will need to pass in an
instance of your Command class to work with in your thread.
"""
class ExampleCommand(sublime_plugin.TextCommand):
def run(self, edit):
exampleThread = ExampleThread(self, edit)
exampleThread.start()
"""
Extend the Thread class and add your functionality in
the run method below.
One thing to remember when moving your code over is
you need to use self.cmd instead of self.
"""
class ExampleThread(threading.Thread):
"""
Remember to pass in the parameters you need
in this thread constructor.
"""
def __init__(self, cmd, edit):
threading.Thread.__init__(self)
self.cmd = cmd
self.edit = edit
"""
Add your functionality here.
If you need to access the main thread, you need to
use sublime.set_timeout(self.callback, 1).
In my example here, you can't call insert text into the editor
unless you are in the main thread.
Luckily that is fast operation.
Basically, time.sleep(3) is a slow operation and will block, hence it
is run in this separate thread.
"""
def run(self):
time.sleep(3)
sublime.set_timeout(self.callback, 1)
"""
This is the callback function that will be called to
insert HelloWorld.
You will probably need to use this to set your status message at
the end. I'm pretty sure that requires that you be on main thread
to work.
"""
def callback(self):
self.cmd.view.insert(self.edit, 0, "Hello, World!")
Update
I found some time integrate your code snippet above using the approach I outlined above. You'll still need to fill in some blanks, but hopefully this gives you an idea of where to put your code. I tested that the basic skeleton still works, which is why the section where you are building the context menu is commented out in this example.
import threading
import time
import sublime
import sublime_plugin
def buildsnippetURL():
return ""
def getsnippets(snippet_url):
time.sleep(3)
return ""
class SyncsnippetsCommand(sublime_plugin.TextCommand):
def run(self, edit):
syncsnippetsThread = SyncsnippetsThread(self, edit)
syncsnippetsThread.start()
class SyncsnippetsThread(threading.Thread):
def __init__(self, cmd, edit):
threading.Thread.__init__(self)
self.cmd = cmd
self.edit = edit
def buildLexerDict(self,snippets):
lexers = snippets[0]['user']['lexers']
lexer_dict = {}
for lexer in lexers:
lexer_dict[lexer] = []
return lexer_dict
def buildsnippetsContextDict(self,snippets,lexer_dict):
snippets_dict = lexer_dict
for snippet in snippets:
snippets_dict[snippet['lexer']].append({"id":str(snippet['id']),
"title":snippet['title']})
return snippets_dict
def run(self):
snippet_url = buildsnippetURL()
snippets_count = 1;
snippets = getsnippets(snippet_url)
"""
context_menu = '['
context_menu += '\n\t{ "caption": "snippets", "id": "file", "children":'
context_menu += '\n\t\t['
if snippets == None:
{"caption":"No snippets available"}
else:
snippets = snippets['objects']
lexers = self.buildLexerDict(snippets)
snippets_dict = self.buildsnippetsContextDict(snippets, lexers)
for j,key in reversed(list(enumerate(reversed(snippets_dict.keys())))):
... loop through JSON and create menu ...
if j == 0:
context_menu += ''
else:
context_menu += ','
context_menu += '\n\t\t]'
context_menu += '\n\t}'
context_menu += '\n]'
f = open(sublime.packages_path() + '\snippetSync\\Context.sublime-menu', 'w')
f.write(context_menu)
f.close
"""
sublime.set_timeout(lambda: self.callback(snippets_count), 1)
def callback(self, snippets_count):
self.cmd.view.set_status('snippet', 'snippet Sync: Added ' + str(snippets_count) + ' snippets from your account.')
sublime.set_timeout(lambda: self.cmd.view.erase_status('snippet'), 3000)