Develop Reference

The publishing code below is stalling on time.sleep() how can i change this behaviour + what would be a good way to implement a thread safe cach

For refreshing skills for a new job i have been trying to implement a very simple price publisher + client + mtm_cache. In the code below in publishing.py the code stops at time.sleep() and doesnt seem to wakeup (say for example two clients connect, but they stop receiving the messages while publisher executes time.sleep()), how can i correct the behavior of the publisher.
Also there is another class MTMCache where i hope to store the MTM sent by the client, would a dictionary be a good thread safe cache which stores list of tuples (time,MTMValue) per client.
As a third point how can i change the implementation of this Publisher class using asyncio?
publishing.py
import threading
import socket
import time
import random
class Publisher(object):
def __init__(self):
self.list_subscribers = []#list of tuples with ip and port
self.mtm_cache = None
self.current_price = 50
self.listening_ports = [1100,1101,1102,1103,1104]
self.stop_threads = False
self.listening_port = 1100
self.sock = None
self.listening = False
#run listening function in another thread
def update_price(self):
tmp = random.randint(-10,10)
#print("Number generated is " + str(tmp))
self.current_price = self.current_price + tmp
def update_price_in_loop(self):
while(True):
self.update_price()
if(self.stop_threads):
break
def send_price_to_subscribers(self):
#in a parallel for loop send current price to subscribers
while(True):
#print(str(self.current_price))
if self.list_subscribers:
for cl in self.list_subscribers:
cl.send(str(self.current_price).encode())
if(self.stop_threads):
break
def start_listener_and_accept_connections(self):
if(not self.listening):
self.sock = socket.socket()
self.sock.bind(('',self.listening_port))
print("sock bound at ", self.listening_port)
while True:
if not self.listening:
print("listening for connections")
self.sock.listen(5)
#above is a blocking call, it is blocking other threads
self.listening = True # how to properly set and utilise this flag
c, addr = self.sock.accept()
print('Got connection from', addr)
self.list_subscribers.append(c)
if(self.stop_threads):
print("closing client connections")
for cl in self.list_subscribers:
cl.close()
self.sock.close()
break
class MTMCache(object):
def __init__(self,publisher):
self.publisher = publisher
self.cache = {} #subscriber token for each of the subscribers in publisher class, currenttime and mtm to be stored here, maybe some other structure besides dict could be used, explore that later
def receive_mtm(self,message):
#message could be string in json format
#have to design this method with appropriate data structures
pass
if __name__ == "__main__":
from threading import *
import time
pub = Publisher()
#pub.update_price()
#print(pub.current_price)
t = Thread(target = pub.update_price_in_loop)
t1 = Thread(target = pub.send_price_to_subscribers)
#t.start()
#t1.start()
t2 = Thread(target = pub.start_listener_and_accept_connections)
t2.start()
t.start()
t1.start()
time.sleep(5)
pub.stop_threads = True
#t2.raise_exception()
#t.raise_exception()
#t1.raise_exception()
t2.join()
t.join()
t1.join()
------
basic_client.py
import socket
s = socket.socket()
port = 1100
s.connect(('127.0.0.1', port))
while(True):
tmp = s.recv(1024).decode()
print(tmp)
if not tmp:
break
s.close()
----
in separate terminals:
python3 publishing.py
term2
python3 basic_client.py
term3
python3 basic_client.py

Is there a way to sync sending information across sockets on a network?

I am programming ada fruit ring LED lights in python, I have setup a local network for a midi controller to read data into my Raspberry Pi 3 and send to the 3 Pi Zero W through web sockets. I am running into the issue where some of the functions I have that run on threads will get out of sync with each other. In other words, one of the LED lights will go at a slightly faster or slower rate and eventually look not synced. I was curious if anyone has run into an issue like this or if I am just using the threads and events the incorrect way.
Here is some of my server side code:
import socket
from socket import *
from threading import Thread
import threading
import board
import neopixel
import sys
import time
import random
import os
from adafruit_led_animation.animation.comet import Comet
def handle_client(client):
p = None
connected = True
lights_on_default()
while True:
global thread_running
data=client.recv(1024).decode()
if not data:
connected = False
thread_running = False
break
if 'Pad' in data:
thread_running = False
thread = threading.Event()
if data == 'Pad 1':
thread_running = False
if p:
p.join()
p = Thread(target=slow_circles, args=(thread,))
p.daemon = True
p.start()
def slow_circles(event):
global thread_running
thread_running = True
while not event.isSet():
if not thread_running:
event.set()
break
for i in range(num_of_pixels):
// goes through the lighting of pixels
if not thread_running:
event.set()
break
pixels.show()
Here is some of my client side code:
import rtmidi.midiutil as midiutil
import time
import socket
buttons = {
36: 'Pad 1',
}
try:
s1 = socket.socket()
s2 = socket.socket()
except:
print('Error making socket')
port1 = 12346
port2 = 12347
serverOne = '192.168.1.18' // 1st Pi Zero W
serverTwo = '192.168.1.17' // 2nd Pi Zero W
def start():
global s1
global s2
notConnected = True
while True:
while notConnected:
connected = False
try:
s1.connect((serverOne, port1))
s2.connect((serverTwo, port2))
connected = True
break
except:
s1.close()
s2.close()
s1.socket.socket()
s2.socket.socket()
if connected:
midiin, port = midiutil.open_midiinput(1)
midiin.set_callback(midiCallback) // gets pad from midi controller
while True:
retry = False
try:
s1.sendall('red connected'.encode('utf8'))
s2.sendall('yellow connected'.encode('utf8'))
except:
retry = True
if retry:
s1.close()
s2.close()
midiin.delete()
break
else:
time.sleep(15)
def midiCallback(message, data):
control = message[0][1]
try:
value = message[0][2]
if (control in button.keys()):
name = buttons[control]
if (value > 0):
return buttonDown(name)
except IndexError:
pass
def buttonDown(button):
s1.sendall(f'{button}'.encode('utf8'))
s2.sendall(f'{button}'.encode('utf8'))
start()
And when I hit the pad and do the function I send data to the two PIs and they start at the same time but over time get out of sync. All help would be appreciated thanks.

Sockets with Python: Listen to Commands to start or stop a loop

I have an alarm that I want to run in a loop. I want to be able to start and stop this loop remotely using sockets. I'm having issues to continuously listen for commands and to start and stop this alarm loop.
There are some similar posts that don't have answers:
How to do a continuous "while loop" in Python with socket listening and waiting every time?
Python socket: trying to receive a new TCP message within a while loop
The problem is that the socket is in a while loop and allows for the alarm loop to start, but now I can't receive the message to stop. How do I continue to listen to start or stop the alarm loop? Below is my attempt to see if there are new data (STOP), but it then stops the alarm loop to listen for incoming messages.
import time
import socket
import sys
mysock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
mysock.bind(("", 1234))
except:
print("Failed to bind")
sys.exit()
mysock.listen(5)
while True:
conn,addr = mysock.accept()
data = conn.recv(1000)
state = b"ALARM OFF"
print(data)
if not data:
break
if data == b'ALARM ON':
state = b"ALARM ON"
print(state)
if data == b'SILENT':
state = b"ALARM OFF"
print(state)
while state == b"ALARM ON":
print("BEEP BEEP BEEP")
conn,addr = mysock.accept()
new_data = conn.recv(1000)
print(new_data)
if new_data:
state == new_data
print(state)
conn.sendall(state)
conn.close()
mysock.close()

I would use a thread to print the message over and over again:
import threading
import time
import socket
import sys
alarm_on = False # Flag to stop the thread
# The thread function
def thread_function():
while alarm_on:
print("BEEP BEEP BEEP")
time.sleep(1)
mysock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
mysock.bind(("", 1234))
except:
print("Failed to bind")
sys.exit()
thrd = None
mysock.listen(5)
while True:
conn,addr = mysock.accept()
while True:
conn,addr = mysock.accept()
data = conn.recv(1000)
state = b"ALARM OFF"
print(data)
if not data:
break
if data == b'ALARM ON':
state = b"ALARM ON"
print(state)
alarm_on = True
# Start the thread
thrd = threading.Thread(target=thread_function)
thrd.start()
if data == b'SILENT':
state = b'ALARM OFF'
print(state)
# Set the flag to stop the thread
alarm_on = False
conn.sendall(state)
conn.close()
mysock.close()

Python socket programming with packets

I am writing a Python based application for which I am using socket programming.
I am following below approach.
Created 1 TCP/IP server, 1 controller TCP/IP client thread and 3 TCP/IP Clients threads.
I want the application to work like this. Whenever controller sends a message it gets broadcast to all 3 TCP/IP clients. Upon received message from controller the client threads perform some task and send data to the server.
Now server has to send this data to controller thread.
The communication part for of the clients and controller is working fine.
Only problem which I am facing is server is putting all the data received from clients to controller socket together.
I want server should put 1 Client thread's data on controller socket, Wait for that data to be picked up. Then place next thread's data.
So far I am using SOCK_STREAM for sockets.
Library:-
#!/usr/bin/python
import select, socket, sys, Queue, errno
usable_port_start = 40000
Internal_ip = "127.0.0.1"
class getTCPports(object):
def __init__(self,starting_port=usable_port_start,address=Internal_ip):
super(getTCPports, self).__init__()
self.IP_address = address
i = 1
delta = 0
while i <= 1:
delta += 2
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
try_port=starting_port + delta
s.bind((self.IP_address,try_port))
self.free_port=try_port
i+=1
except socket.error as e:
if e.errno == errno.EADDRINUSE:
print("Port" , try_port , "is already in use")
else:
# something else raised the socket.error exception
print(e)
s.close()
class IPCLib(getTCPports):
server_port = 0
controller_port = 0
client_map = {}
def __init__(self):
super(IPCLib,self).__init__()
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind((self.IP_address, self.free_port))
self.inputs = [self.socket]
self.is_alive = True
self.broadcast_list = []
#classmethod
def modify_server_port(cls,port):
cls.server_port = port
#classmethod
def modify_client_port(cls,identity,port):
if 0 <= identity.find("CONTROLLER"):
cls.controller_port = port
elif 0 <= identity.find("CLIENT"):
cls.client_map[identity] = port
def start_TCP_server(self):
self.socket.setblocking(0)
self.socket.listen(10)
self.modify_server_port(self.free_port)
while self.is_alive:
inputready,outputready,exceptready = select.select(self.inputs,[],[])
for s in inputready: #check each socket that select() said has available data
if s == self.socket: #if select returns our server socket, there is a new
#remote socket trying to connect
client, address = s.accept()
self.inputs.append(client) #add it to the socket list so we can check it now
self.broadcast_list.append(client)
#print 'new client added%s'%str(address)
else:
# select has indicated that these sockets have data available to recv
data = s.recv(4096)
if data:
#print '%s Received From Client(on server)-> %s'%(data,s.getpeername()[1])
#Uncomment below to echo the recv'd data back
#to the sender... loopback!
if s.getpeername()[1]==self.controller_port:
self.broadcast(data)
else: #if sender is monitoring clients, send data to only controller
self.send_to_controller(data)
else:#if recv() returned NULL, that usually means the sender wants
#to close the socket.
s.close()
self.inputs.remove(s)
#if running is ever set to zero, we will call this
server.close()
def start_TCP_client(self,identity):
self.modify_client_port(identity,self.free_port)
self.socket.connect((self.IP_address,self.server_port))
def stop_TCP_client(self):
self.socket.shutdown(socket.SHUT_RDWR)
self.socket.close()
def broadcast(self,message):
for client in self.broadcast_list:
if client.getpeername()[1]!=self.controller_port:
try:
client.send(message)
except:
client.close()
# if the link is broken, we remove the client
remove(clients)
def send_to_controller(self,message):
for client in self.broadcast_list:
if client.getpeername()[1]==self.controller_port:
try:
client.send(message)
except:
client.close()
# if the link is broken, we remove the client
remove(clients)
def send_data(self,data):
self.socket.send(data)
def receive_data(self):
message = self.socket.recv(4096)
return message
Driver Program:-
#!/usr/bin/python
from IPCLib import *
import threading
import os
import time
def run_server():
i1=IPCLib()
print("Task assigned to thread: {}".format(threading.current_thread().name))
print("ID of process running task: {}".format(os.getpid()))
i1.start_TCP_server()
def run_controller(identity):
i1=IPCLib()
print("Task assigned to thread: {}".format(threading.current_thread().name))
print("ID of process running task: {}".format(os.getpid()))
i1.start_TCP_client(identity)
print("server port " , i1.server_port)
print("controller port", i1.controller_port)
print("bsc info", i1.client_map)
time.sleep(1)
while i1.is_alive:
i1.send_data("hello")
print"Next Clock"
sender_map={}
sender_list = []
sender_list = i1.client_map.values()
for sender in sender_list:
sender_map[sender] = False
i=1
#print any(sender_map.values())
while any(value == False for value in sender_map.values()):
print("Loop Iteration %s"%i)
data = i1.receive_data()
temp = data.split(",")
port = temp.pop(0)
sender_map[int(port)] = True
data = ",".join(temp)
print("Data %s received from port %s"%(data,port))
print sender_map
i+=1
print sender_list
time.sleep(1)
i1.stop_TCP_client()
def run_monitors(identity):
i1=IPCLib()
print("Task assigned to thread: {}".format(threading.current_thread().name))
print("ID of process running task: {}".format(os.getpid()))
i1.start_TCP_client(identity)
print("server port " , i1.server_port)
print("controller port", i1.controller_port)
print("bsc info", i1.client_map)
while i1.is_alive:
if i1.receive_data():
output = "%d"%i1.free_port
output = output + "," + "Hello"
i1.send_data(output)
i1.stop_TCP_client()
# creating thread
t1 = threading.Thread(target=run_server, name='server')
t3 = threading.Thread(target=run_monitors, name='Client1',args=("CLIENT-1",))
t4 = threading.Thread(target=run_monitors, name='Client2',args=("CLIENT-2",))
t5 = threading.Thread(target=run_monitors, name='Client3',args=("CLIENT-3",))
t6 = threading.Thread(target=run_monitors, name='Clinet4',args=("CLIENT-4",))
#make threads deamons
t1.daemon = True
t3.daemon = True
t4.daemon = True
t5.daemon = True
t6.daemon = True
# starting threads
try:
t1.start()
time.sleep(0.1)
t3.start()
time.sleep(0.1)
t4.start()
time.sleep(0.1)
t5.start()
time.sleep(0.1)
t6.start()
time.sleep(0.1)
run_controller("CONTROLLER")
except KeyboardInterrupt:
t1.is_alive = False
t3.is_alive = False
t4.is_alive = False
t5.is_alive = False
t6.is_alive = False
How can I force server to wait till the time there is already some data on the socket?

python Client hangs when no data to receive from server and hangs in that thread w/o letting client send

I am trying to figure out how to get my client to send and receive data 'simultaneously' and am using threads. My problem is that, depending on the way I set it up, the way here it waits for data from the server in the recieveFromServer function which is in its own thread and cannot stop it when nothing will be sent. The other way it just waits for user input, and will send to the server and then I'd call the function recieveFromServer after the client sends a message to the server which doesn't allow for fluent communication, but cannot get it to alternate automatically. How do I release the thread when the client has nothing to be sent, or there is no more to be received from the server.
It would get to long if I tried to explain everything I have tried. :)
Thanks.
The client:
from socket import *
from threading import *
import thread
import time
from struct import pack,unpack
from networklingo import *
#from exception import *
HOST = '192.168.0.105'
PORT = 21567
BUFFSIZE = 1024
ADDR = (HOST,PORT)
lock = thread.allocate_lock()
class TronClient:
def __init__(self,control=None):
self.tcpSock = socket(AF_INET,SOCK_STREAM)
#self.tcpSock.settimeout(.2)
self.recvBuff = []
def connect(self):
self.tcpSock.connect(ADDR)
self.clientUID = self.tcpSock.recv(BUFFSIZE)
print 'My clientUID is ', self.clientUID
t = Thread(target = self.receiveFromSrv())
t.setDaemon(1)
t.start()
print 'going to main loop'
self.mainLoop()
#t = Thread(target = self.mainLoop())
#t.setName('mainLoop')
#t.setDaemon(1)
#t.start()
def receiveFromSrv(self):
RECIEVING = 1
while RECIEVING:
#print 'Attempting to retrieve more data'
#lock.acquire()
#print 'Lock Aquired in recieveFromSrv'
#try:
data = self.tcpSock.recv(BUFFSIZE)
#except socket.timeout,e:
#print 'Error recieving data, ',e
#continue
#print data
if not data: continue
header = data[:6]
msgType,msgLength,clientID = unpack("hhh",header)
print msgType
print msgLength
print clientID,'\n'
msg = data[6:]
while len(msg) < msgLength:
data = self.tcpSock.recv(BUFFSIZE)
dataLen = len(data)
if dataLen <= msgLength:
msg += data
else:
remLen = msgLength-len(data) #we just need to retrieve first bit of data to complete msg
msg += data[:remLen]
self.recvBuff.append(data[remLen:])
print msg
#else:
#lock.release()
# print 'lock release in receiveFromSrv'
#time.sleep(2)
#RECIEVING = 0
def disconnect(self,data=''):
self.send(DISCONNECT_REQUEST,data)
#self.tcpSock.close()
def send(self,msgType,msg):
header = pack("hhh",msgType,len(msg),self.clientUID)
msg = header+msg
self.tcpSock.send(msg)
def mainLoop(self):
while 1:
try:
#lock.acquire()
#print 'lock aquired in mainLoop'
data = raw_input('> ')
except EOFError: # enter key hit without any data (blank line) so ignore and continue
continue
#if not data or data == '': # no valid data so just continue
# continue
if data=='exit': # client wants to disconnect, so send request to server
self.disconnect()
break
else:
self.send(TRON_CHAT,data)
#lock.release()
#print 'lock released in main loop'
#self.recieveFromSrv()
#data = self.tcpSock.recv(BUFFSIZE)
#t = Thread(target = self.receiveFromSrv())
#t.setDaemon(1)
#t.start()
if __name__ == "__main__":
cli = TronClient()
cli.connect()
#t = Thread(target = cli.connect())
#t.setName('connect')
#t.setDaemon(1)
#t.start()
The server (uses a lock when incrementing or decrementing number of clients):
from socket import *
from threading import *
import thread
from controller import *
from networklingo import *
from struct import pack,unpack
HOST = ''
PORT = 21567
BUFSIZE = 1024
ADDR = (HOST,PORT)
nclntlock = thread.allocate_lock()
class TronServer:
def __init__(self,maxConnect=4,control=None):
self.servSock = socket(AF_INET,SOCK_STREAM)
# ensure that you can restart server quickly when it terminates
self.servSock.setsockopt(SOL_SOCKET,SO_REUSEADDR,1)
self.servSock.bind(ADDR)
self.servSock.listen(maxConnect)
# keep track of number of connected clients
self.clientsConnected = 0
# give each client a unique identfier for this run of server
self.clientUID = 0
# list of all clients to cycle through for sending
self.allClients = {}
# keep track of threads
self.cliThreads = {}
#reference back to controller
self.controller = control
self.recvBuff = []
def removeClient(self,clientID,addr):
if clientID in self.allClients.keys():
self.allClients[clientID].close()
print "Disconnected from", addr
nclntlock.acquire()
self.clientsConnected -= 1
nclntlock.release()
del self.allClients[clientID]
else:
print 'ClientID is not valid'
def recieve(self,clientsock,addr):
RECIEVING = 1
# loop serving the new client
while RECIEVING: # while PLAYING???
try:
data = clientsock.recv(BUFSIZE)
except:
RECIEVING = 0
continue
# if not data: break #no data was recieved
if data != '':
print 'Recieved msg from client: ',data
header = data[:6]
msgType,msgLength,clientID = unpack("hhh",header)
print msgType
print msgLength
print clientID,'\n'
if msgType == DISCONNECT_REQUEST: #handle disconnect request
self.removeClient(clientID,addr)
else: #pass message type and message off to controller
msg = data[6:]
while len(msg) < msgLength:
data = self.tcpSock.recv(BUFSIZE)
dataLen = len(data)
if dataLen <= msgLength:
msg += data
else:
remLen = msgLength-len(data) #we just need to retrieve first bit of data to complete msg
msg += data[:remLen]
self.recvBuff.append(data[remLen:])
print msg
# echo back the same data you just recieved
#clientsock.sendall(data)
self.send(TRON_CHAT,msg,-1) #send to client 0
for k in self.allClients.keys():
if self.allClients[k] == clientsock:
self.removeClient(k,addr)
print 'deleted after hard exit from clientID ', k
#self.cliThreads[k].join()
#del self.cliThreads[k]
# then tell controller to delete player with k
break
def send(self,msgType,msg,clientID=-1):
header = pack("hhh",msgType,len(msg),clientID)
msg = header+msg
if clientID in self.allClients:
self.allClients[clientID].send(msg)
elif clientID==ALL_PLAYERS:
for k in self.allClients.keys():
self.allClients[k].send(msg)
def mainLoop(self):
global nclntlock
try:
while self.controller != None and self.controller.state == WAITING:
print 'awaiting connections'
clientsock, caddy = self.servSock.accept()
nclntlock.acquire()
self.clientsConnected += 1
nclntlock.release()
print 'Client ',self.clientUID,' connected from:',caddy
clientsock.setblocking(0)
clientsock.send(str(self.clientUID))
self.allClients[self.clientUID] = clientsock
t = Thread(target = self.recieve, args = [clientsock,caddy])
t.setName('recieve-' + str(self.clientUID))
self.cliThreads[self.clientUID] = t
self.clientUID += 1
# t.setDaemon(1)
t.start()
finally:
self.servSock.close()
if __name__ == "__main__":
serv = TronServer(control = LocalController(nPlayers = 3, fWidth = 70, fHeight = 10))
t = Thread(target = serv.mainLoop())
t.setName('mainLoop')
# t.setDaemon(1)
t.start()

I think you want to try and set the socket to non-blocking mode:
http://docs.python.org/library/socket.html#socket.socket.setblocking
Set blocking or non-blocking mode of
the socket: if flag is 0, the socket
is set to non-blocking, else to
blocking mode. Initially all sockets
are in blocking mode. In non-blocking
mode, if a recv() call doesn’t find
any data, or if a send() call can’t
immediately dispose of the data, a
error exception is raised; in blocking
mode, the calls block until they can
proceed. s.setblocking(0) is
equivalent to s.settimeout(0);
s.setblocking(1) is equivalent to
s.settimeout(None).
Also, instead of using raw sockets, have you considdered using the multiprocessing module. It is a higher-level abstraction for doing network IO. The section on Pipes & Queues is specific to sending and receiving data between a client/server.