Develop Reference

The publishing code below is stalling on time.sleep() how can i change this behaviour + what would be a good way to implement a thread safe cach

For refreshing skills for a new job i have been trying to implement a very simple price publisher + client + mtm_cache. In the code below in publishing.py the code stops at time.sleep() and doesnt seem to wakeup (say for example two clients connect, but they stop receiving the messages while publisher executes time.sleep()), how can i correct the behavior of the publisher.
Also there is another class MTMCache where i hope to store the MTM sent by the client, would a dictionary be a good thread safe cache which stores list of tuples (time,MTMValue) per client.
As a third point how can i change the implementation of this Publisher class using asyncio?
publishing.py
import threading
import socket
import time
import random
class Publisher(object):
def __init__(self):
self.list_subscribers = []#list of tuples with ip and port
self.mtm_cache = None
self.current_price = 50
self.listening_ports = [1100,1101,1102,1103,1104]
self.stop_threads = False
self.listening_port = 1100
self.sock = None
self.listening = False
#run listening function in another thread
def update_price(self):
tmp = random.randint(-10,10)
#print("Number generated is " + str(tmp))
self.current_price = self.current_price + tmp
def update_price_in_loop(self):
while(True):
self.update_price()
if(self.stop_threads):
break
def send_price_to_subscribers(self):
#in a parallel for loop send current price to subscribers
while(True):
#print(str(self.current_price))
if self.list_subscribers:
for cl in self.list_subscribers:
cl.send(str(self.current_price).encode())
if(self.stop_threads):
break
def start_listener_and_accept_connections(self):
if(not self.listening):
self.sock = socket.socket()
self.sock.bind(('',self.listening_port))
print("sock bound at ", self.listening_port)
while True:
if not self.listening:
print("listening for connections")
self.sock.listen(5)
#above is a blocking call, it is blocking other threads
self.listening = True # how to properly set and utilise this flag
c, addr = self.sock.accept()
print('Got connection from', addr)
self.list_subscribers.append(c)
if(self.stop_threads):
print("closing client connections")
for cl in self.list_subscribers:
cl.close()
self.sock.close()
break
class MTMCache(object):
def __init__(self,publisher):
self.publisher = publisher
self.cache = {} #subscriber token for each of the subscribers in publisher class, currenttime and mtm to be stored here, maybe some other structure besides dict could be used, explore that later
def receive_mtm(self,message):
#message could be string in json format
#have to design this method with appropriate data structures
pass
if __name__ == "__main__":
from threading import *
import time
pub = Publisher()
#pub.update_price()
#print(pub.current_price)
t = Thread(target = pub.update_price_in_loop)
t1 = Thread(target = pub.send_price_to_subscribers)
#t.start()
#t1.start()
t2 = Thread(target = pub.start_listener_and_accept_connections)
t2.start()
t.start()
t1.start()
time.sleep(5)
pub.stop_threads = True
#t2.raise_exception()
#t.raise_exception()
#t1.raise_exception()
t2.join()
t.join()
t1.join()
------
basic_client.py
import socket
s = socket.socket()
port = 1100
s.connect(('127.0.0.1', port))
while(True):
tmp = s.recv(1024).decode()
print(tmp)
if not tmp:
break
s.close()
----
in separate terminals:
python3 publishing.py
term2
python3 basic_client.py
term3
python3 basic_client.py

is the logic behind my trivial tftp client flawed?

I'm still a student so I might have missed something really obvious.
So I am stressing out about this so much.
But anyway here's my TFTP python code, all it does is downloading a text-file from our lecturer's server.
The file generated by it looks like this:
http://pastebin.com/TP8hngxM
And the original file like this:
http://pastebin.com/xDMjkABp
And if you run it through a difference checker, the difference is miniscule and in only 1 specific spot and it's really difficult for me to figure out why is this happening.
The downloaded file has a few extra words.
If you have spare 5 minutes, could you please check out my nested while loop (everything else was provided by the lecturer and can't be changed) to see if there's anything wrong with it?
The worst thing is that I've already had it working, but I lost my memory stick and I lost the most up-to-date version of the program that was running 100% fine.
So as I said, it's only about the nested while loop, I' not allowed to change anything above it.
#!/usr/bin/python
import struct
import sys
import os
import select
import glamnetsim
from socket import *
serverHost = 'mcgreg.comp.glam.ac.uk'
serverPort = 69
timeoutSecs = 5
debugging = False
RRQ, WRQ, DATA, ACK, ERROR = range(1, 6)
codeDescriptions = {RRQ:"RRQ", WRQ:"WRQ", DATA:"DATA", ACK:"ACK", ERROR:"ERROR"}
def printf (format, *args):
print str(format) % args,
def finish ():
printf("you should see\n1e951df315d433aa4df2065a1ad31311\n")
os.system("md5sum newfile.txt")
sys.exit(0)
def sendPacket (packet, port):
global sock, debugging
global serverIp
if debugging:
for i in packet:
print ('%02x' % ord(i)),
print ''
sock.sendto(packet, (serverIp, port))
def sendReadRequest (filename, mode):
global serverPort
format = "!H%ds" % (len(filename)+1)
format += "%ds" % (len(mode)+1)
s = struct.pack(format, 1, filename, mode)
sendPacket(s, serverPort)
def sendRealAck(blockno, port):
s = struct.pack("!H", 4)
s += struct.pack("!H", blockno)
sendPacket(s, port)
def sendACK (blockno, port):
print " -> ACK:%d\n" % blockno
if blockno == 0:
sendReadRequest("the_machine_stops.txt", "octet")
else:
sendRealAck(blockno, port)
def stripPacket (s):
if len(s)>3:
code = struct.unpack("!H", s[:2])[0]
blockno = struct.unpack("!H", s[2:4])[0]
data = s[4:]
code, data = glamnetsim.simulatePossibleError (code, data)
return code,blockno,data
else:
debugPrint("corrupt packet")
return -1,-1,""
def debugPrint (s):
global debugging
if debugging:
print s
def getDesc (c):
global codeDescriptions
return codeDescriptions[c]
sock = socket(AF_INET, SOCK_DGRAM)
serverIp = gethostbyname(serverHost)
sock.setblocking(1)
sendReadRequest("the_machine_stops.txt", "netascii")
lastblock = 0
blockno = 0
port = serverPort
f = open("newfile.txt", "w")
while True:
while True:
if blockno == lastblock+1:
break
r, w, x = select.select([sock], [], [], 5.0)
if r == []:
sendACK(lastblock, port)
else:
(packet, (address, port)) = sock.recvfrom(512+4)
code, newblock, text = stripPacket(packet)
print code, blockno
if code is 3:
blockno = newblock
sendACK(blockno, port)
if code is 5:
sendACK(lastblock, port)
print "Bn: " + str(blockno) + " Lb: " + str(lastblock)
lastblock = blockno
f.write(text)
print "OK"
if len(text) < 512:
break
f.close()
finish()

f.write(text)
That line is run with a stale value if blockno == lastblock+1. It probably should be within the inner loop.

How to do exception handling on multiprocessing python

I use multiprocessing in my python code.
My python code imports Pysnmp and multiprocessing.
Half of the time my code runs smoothly.
But unfortunately half of the other time my code doesn't work and shows the exception "pyasn1.error.pyasn1 error: type tagset".
My code first creates a "multiprocessing.dummy.Pool(numOfThreads)" with a number of threads.
Then it calls "p.map(sendSNMPGet, [ipRange + '.' + str(x) for x in range(1,256)])" which takes the function "sendSNMPGet" as the thread function, and calls the function 255 times with the values of "1-255".
This is my code:
from pysnmp.entity.rfc3413.oneliner import cmdgen
from pysnmp.proto.rfc1902 import Integer, IpAddress, OctetString
import multiprocessing.dummy
import multiprocessing
def SNMPGet(ip, community, oid, version = 1):
generator = cmdgen.CommandGenerator()
comm_data = cmdgen.CommunityData('server', community, version) # 1 means version SNMP v2c
transport = cmdgen.UdpTransportTarget((ip, 161),timeout=0.5,retries=2)
real_fun = getattr(generator, 'getCmd')
res = (errorIndication, errorStatus, errorIndex, varBinds)\
= real_fun(comm_data, transport, oid)
if (errorStatus == 2): #Is there an EndOfMib() Error
return
elif (errorIndication is None or errorStatus is True):
with open("Maprinter.txt", "a") as myfile: myfile.write("Date is: " + datetime.datetime.now().strftime("%d-%m-%Y %H:%M:%S") + " IP is: " + ip + ", Response: %s\n" % varBinds)
print("IP is: " + ip + ", Response: %s\n" % varBinds)
return
def sendSNMPGet(ip):
return SNMPGet(ip, 'public', '1.3.6.1.2.1.43.5.1.1.1.1', 1)
def snmp_range(ipRange, start, end):
num_threads = 4 * multiprocessing.cpu_count()
p = multiprocessing.dummy.Pool(num_threads)
p.map(sendSNMPGet, [ipRange + '.' + str(x) for x in range(start,end)])
def GetIpsRange(ips):
print('starting scan')
if __name__ == "__main__":
if(ips != ''): #if there are ips in registry
for ip in range(len(ips)):
snmp_range(ips[ip], 1, 256)
print('ending scan')
GetIpsRange('10.0.0')
Despite the use of Pysnmp I think my exception occurs due to lack of proper coding of multiprocessing.
How could I change my code to handle this exception or even better, prevent it from happening?

python threads exits immediately

I am new to python.
I am trying out Hbase thrift client using thrift. I got some code on net, which I just modify to work with latest version of thrift but when I run the code , it just exit, no threads are started.
Here is the code.
import json, traceback, sys, datetime, time, logging, threading, random
import logging.handlers
import thrift
sys.path.append('gen-py')
from thrift.transport.TSocket import TSocket
from thrift.transport.TTransport import TBufferedTransport
from thrift.protocol import TBinaryProtocol
from hbase import THBaseService
gWritenItems = 0
gStartT = 0
gEndT = 0
recordsPerBatch = 300 #reports per client per day
columns = 3
#config
concurrent = 10
records = 60000#6000000 #6 million
bytesPerRecord = 1024
mylock = threading.RLock()
class writeThread(threading.Thread):
def __init__(self, threadname, RecordsThreadwillwrite):
threading.Thread.__init__(self, name = threadname)
bytesPerColumn = int(bytesPerRecord/columns) - 11 #suppose 3 columns
self.columnvalue = "value_" + "x"*bytesPerColumn + "_endv"
self.tbwBatch = int (RecordsThreadwillwrite / recordsPerBatch)
self.transport = TBufferedTransport(TSocket('pnq-adongrevm1', 5151), 40960)
self.transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = THBaseService.Client(protocol)
self.table = "example"
def run(self):
print "+%s start" % (self.getName())
global gEndT
global gWritenItems
threadWritenItem = 0
for loopidx in xrange(0, self.tbwBatch):
self.write_hbase() #write
threadWritenItem += recordsPerBatch
mylock.acquire()
gEndT = time.time()
gWritenItems += threadWritenItem
print "%s done, %s seconds past, %d reocrds saved" % (self.getName(), gEndT-gStartT, gWritenItems)
mylock.release()
self.transport.close()
def write_hbase(self): #write 50 rowkyes, and 3 column families in each rowkey
print self.getName(), "Start write"
batchmutations = []
for i in xrange(0, recordsPerBatch): # write to db, 300 items together
mutations = []
rowkey = "RK_%s_%s" % (random.random(), time.time())
for ii in xrange(0, columns):
mutations.append(THBaseService.TPut(row=rowkey, columnValues=[TColumnValue(family="f1", qualifier="%s"%ii, value=self.columnvalue)]))
self.client.putMultiple(self.table,mutations)
itemsPerThread = int(records / concurrent)
for threadid in xrange(0, concurrent):
gStartT = time.time()
t = writeThread("Thread_%s" % threadid, itemsPerThread)
t.start();
print "%d thread created, each thread will write %d records" % (concurrent, itemsPerThread)
I just get a message 10 thread created, each thread will write 6000 records

Yep, this is because you are not waiting for threads to finish their job, so the main thread just exits. Try this:
itemsPerThread = int(records / concurrent)
threads = []
for threadid in xrange(0, concurrent):
gStartT = time.time()
t = writeThread("Thread_%s" % threadid, itemsPerThread)
t.start();
threads.append(t)
# wait until all finish the job
for t in threads:
t.join()
EDIT Ha, I don't think I'm right here, because you didn't mark your threads as daemons. It should work even without joining. But have a look at this code:
class CustomThread(threading.Thread):
def run(self):
print "test"
for x in xrange(0, 10):
t = CustomThread()
t.start()
It will always reach print "test" line no matter what. So in your code it should always reach print "+%s start" % (self.getName()) no matter what. Are you sure it doesn't work? :)
If it doesn't, then there are only two possibilities:
There is a blocking operation and/or exception in your __init__ method. But then it would not reach final print;
concurrent variable is 0 for some reason (which is not consistent with the final print).

Python pyserial and runaway CPU usage

MacOS 10.7.3, python 2.5
I am using pyserial to open a connection to an external server. The connection is opened as:
HOST = '10.0.0.1'
PORT = '16010'
theURL = 'socket://' + HOST + ':' + PORT
conn = serial.serial_for_url(theURL, baudrate=115200)
conn.timeout = 2
and then the read looks like this:
try:
while len(rawData) == 0 and self.shutdown == False:
rawData = conn.readline()
except:
some error handling code...
The problem is, if I kill the server at 10.0.0.1:16010 the code keeps running, but the cpu usage goes to 100%. No error is thrown, so the except is never entered.
This looks like a problem in pyserial, but maybe someone here has run into this before and knows how to detect the lost connection so the situation can be handled gracefully.
Thanks.

If you're not depending on .readline() you could do it like this:
self.plugin.conn = Serial(..., timeout = 1)
...
if not self.shutdown:
rawData = self.plugin.conn.read(1)
if rawData:
rawData += self.plugin.conn.read(self.plugin.conn.inWaiting())
else:
raise Exception("timeout")
I'm not sure if I got your intent right so you might have to adjust...

The fact that your CPU usage is pegged probably indicates the the readline call is not blocking to timeout but returns instantly. So if your normal timeout
is 2, you could use:
from time import time
try:
while len(rawData) == 0 and self.shutdown == False:
before = time()
rawData = conn.readline()
if (len(rawData)==0) and ((time()-before)<2):
raise Exception("Early readline return.")
except:
some error handling code...

A very good solution to this can be found here:
class ReadLine:
def __init__(self, s):
self.buf = bytearray()
self.s = s
def readline(self):
i = self.buf.find(b"\n")
if i >= 0:
r = self.buf[:i+1]
self.buf = self.buf[i+1:]
return r
while True:
i = max(1, min(2048, self.s.in_waiting))
data = self.s.read(i)
i = data.find(b"\n")
if i >= 0:
r = self.buf + data[:i+1]
self.buf[0:] = data[i+1:]
return r
else:
self.buf.extend(data)
ser = serial.Serial('COM7', 9600)
rl = ReadLine(ser)
while True:
print(rl.readline())