One process is receiving data from a socket and then putting it in a queue, and the other one is processing the queued data. How to make them both run at the same time?
This socket is serve_forever while the processing of data will only run if the queue is not empty.
i have it working. i don't know if this is THE ANSWER. but it's working, well, so far. maybe there are or there will be bugs (hopefully none). any suggestions for improvements is welcome.
import multiprocessing
import socket
from multiprocessing import Process, Queue
import time
def handle(connection, address):
try:
while True:
data = connection.recv(1024)
if data == "":
break
else :
print "RECEIVE DATA : " + str(data)
xdata = data.strip()
xdata = data.split(" ")
for xd in xdata :
print "PUT Task : " + str(xd)
QueueTask.put((xd), block=True, timeout=5)
connection.sendall(data)
except:
print "Problem handling request"
finally:
connection.close()
class Server(object):
def __init__(self, hostname, port):
self.hostname = hostname
self.port = port
def start(self):
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.bind((self.hostname, self.port))
self.socket.listen(1)
while True:
conn, address = self.socket.accept()
process = multiprocessing.Process(target=handle, args=(conn, address))
process.daemon = True
process.start()
def f_Processor():
time.sleep(10)
print 'PROCESSOR Starting'
while 1:
try :
job = QueueTask.get(True,1)
print "GET Task : " + str(job)
time.sleep(5)
except Exception as err :
pass
print 'PROCESSOR Exiting'
if __name__ == "__main__":
server = Server("localhost", 9999)
QueueTask = Queue()
try:
p = multiprocessing.Process(name='Processing', target=f_Processor)
p.start()
server.start()
p.join()
except:
print "Unexpected exception"
finally:
for process in multiprocessing.active_children():
process.terminate()
process.join()
print "All done"
Also it's depend, if you have a server or client application. If it is a server than you can use
SocketServer.TCPServer.allow_reuse_address = True
self.server = TCPFactory( ( HOST, PORT ), TCPRequestHandler, params )
# Start a thread with the server
self.server_thread = threading.Thread( target = self.server.serve_forever )
self.server_thread.setDaemon( True )
self.server_thread.start()
class TCPFactory( SocketServer.ThreadingTCPServer ):
def __init__( self, server_address, RequestHandlerClass, params ):
"""
"""
SocketServer.ThreadingTCPServer.__init__( self, server_address, RequestHandlerClass )
self.patrams = params
class TCPRequestHandler( SocketServer.BaseRequestHandler ):
def setup( self ):
print self.server.params
pass
def handle( self ):
pass
So if a client connected to the server it will start a new thread. The setup and
handler function will be called automatically.
For the other thread you can use a timer, or an other thread
myt = Timer( 2, chackque, () )
myt.start()
def chackque():
if not myq.empty():
#Do what you want
or just start an other thread:
mythread = threading.Thread( target = chackque, args = ( myargs, ) )
mythread.setDaemon( True )
mythread.start()
def chackque():
while True:
if not myq.empty():
#Do what you want
Related
From my understanding python can only run 1 thread at a time so if I were to do something like this
import socket, select
from threading import Thread
import config
class Source(Thread):
def __init__(self):
self._wait = False
self._host = (config.HOST, config.PORT + 1)
self._socket = socket.socket()
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._sock = None
self._connections = []
self._mount = "None"
self._writers = []
self._createServer()
Thread.__init__(self)
def _createServer(self):
self._socket.bind(self._host)
self._socket.listen(2)
self._connections.append(self._socket)
self._audioPackets=[]
def _addPacket(self, packet):
self._audioPackets.append(packet)
def _removePacket(self, packet):
self._audioPackets.remove(packet)
def _getPacket(self):
if len(self._audioPackets) > 0:
return self._audioPackets[0]
else:
return None
def _sendOK(self, sock):
sock.send("OK")
def _sendDenied(self, sock):
sock.send("DENIED")
def _sendMount(self, sock):
sock.send("mount:{0}".format(self._mount))
def _sendBufPacket(self, sock, packet):
packet = "buffer:%s" % packet
sock.send(packet)
def recv(self, sock, data):
data = data.split(":", 1)
if data[0] == "WAIT": self._wait = True
elif data[0] == "STOP_WAITING": self._wait = False
elif data[0] == "LOGIN":
if data[1] == config.SOURCE_AUTH:
self._source = sock
self._sendOK(sock)
else:
self._sendClose(sock)
elif data[0] == "MOUNT":
if self._source == sock:
self._mount = data[1]
else:
self._sendClose(sock)
elif data[0] == "CLIENT":
self._sendMount(sock)
self._writers.append(sock)
def _sendCloseAll(self):
for sock in self._connections:
sock.send("CLOSE")
sock.close()
def _sendClose(self, sock):
sock.send("CLOSE")
sock.close()
def main(self):
while True:
rl, wl, xl = select.select(self._connections, self._writers, [], 0.2)
for sock in rl:
if sock == self._socket:
con, ip = sock.accept()
self._connections.append(con)
else:
data = sock.recv(config.BUFFER)
if data:
self.recv(sock, data)
else:
if sock in self._writers:
self._writers.remove(sock)
if sock in self._connections:
self._connections.remove(sock)
for sock in wl:
packet = self._getPacket()
if packet != None:
self._sendBufPacket(sock, packet)
def run(self):
self.main()
class writeThread(Thread):
def __init__(self):
self.running = False
def make(self, client):
self.client = client
self.running = True
def run(self):
host = (config.HOST, config.PORT+1)
sock = socket.socket()
sock.connect(host)
sock.send("CLIENT")
sock.send("MOUNT:mountpoint")
while self.running:
data = sock.recv(config.BUFFER)
if data:
data = data.split(":", 1)
if data[0] == "buffer":
self.client.send(data[1])
elif data[0] == "CLOSE":
self.client.close()
break
if __name__=="__main__":
source = Source()
source.start()
webserver = WebServer()
webserver.runloop()
if I need to build the webserver part I will. But, I'll explain it.
Okay, so basically when someone connects to the websever under the mountpoint that was set, They will get there own personal thread that then grabs the data from Source() and sends it to them. Now say another person connects to the mount point and the last client as well as the source is still going. Wouldn't the new client be blocked from getting the Source data considering there are two active threads?
Your understanding of how Threads work in Python seems to be incorrect, based on the question you are asking. If used correctly, threads will not be blocking: you can instantiate multiple thread with Python. The limitation is that, due to the Global Interpreter Lock (GIL), you cannot get the full parallelism expected in thread programming (e.g. simultaneous execution and thus, reduced runtime).
What is going to happen in your case is that the two threads will take, together, the same amount of time that they would take if they were executed sequentially (although that is not necessarily what happens in practice).
Okay, I have copy and pasted some Python3 code that I have already written for a project that I am currently working on. With modification, you can make this code serve your purposes.
The code uses multiprocessing and multithreading. For my purposes, I am using multiprocessing so that sockets will run on one processor, and I can run a GUI program on another processor. You can remove the multiprocessor part if you prefer. The code below runs a socket message server. The server will listen for clients one at a time. Once a client has connected, a new thread will be initiated to handle all the communications between the server and each client. The server will then continue to search for for clients. At the moment however, the server only listens to data being sent from each client, and then it prints it to the terminal. With a small amount of effort, you can modify my code to send information from the server to each client individually.
import multiprocessing
import threading
from threading import Thread
class ThreadedServer(object):
def __init__(self, host, port):
self.host = host
self.port = port
self.sock = socket(AF_INET, SOCK_STREAM)
self.sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self.sock.bind((self.host, self.port))
def listen(self):
self.sock.listen(3) #Allow 3 Clients to connect to this server
while True:
#The program will search for one client at a time
print("Searching for Client")
client, address = self.sock.accept()
print(address, " is connected")
#client.settimeout(60)
#Once a client has been found, start a individual client thread
d = threading.Thread(target = self.listenToClient, args=(client, address))
d.daemon = True
d.start()
def listenToClient(self, client, address):
size = 1024
while True:
try:
data = client.recv(size)
if not data:
break
if data:
print(data)
#client.send(response)
else:
raise error('Client disconnected')
except:
client.close()
return False
def dataSharingHost():
#Using Sockets to send information between Processes
#This is the server Function
#ThreadServer(Host_IP, Port_Number), for LocalHost use ''
ThreadedServer('', 8000).listen()
def Main():
commServer = multiprocessing.Process(target=dataSharingHost, args=())
commServer.daemon = True
commServer.start()
if __name__== '__main__':
Main()
And to be fair, my code is modified from https://www.youtube.com/watch?v=qELZAi4yra8 . The client code is covered in those videos. I think the 3rd video covers the multiple client connects.
I wrote a udp no echo server in my program, I used thread to running and listen some message send by others. But it seems I couldn't stop it use tl.stop(), when I input q or quit. Some of my code is following:
class treadListen(threading.Thread):
def __init__(self):
self.running = True
threading.Thread.__init__(self)
def run(self):
address = ('localhost', 16666)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind(address)
while True:
data = sock.recv(65535)
print "MESSAGE:{0}".format(data)
sock.close()
def stop(self):
self.running = False
# end of class thread_clock
if __name__ == "__main__":
tl = treadListen()
tl.start()
while True:
message = raw_input("CMD>")
if not message:
print "Please input command!"
continue
elif (message == 'quit') or (message == 'q'):
tl.stop()
break
else:
print "input is {0}".format(message)
# do something
continue
print "[CONNECTION CLOSED!]"
I was trying to add sock.shutdown(socket.SHUT_RDWR) and sock.close() to def stop of class, but it doesn't work.
How can I stop the thread safety? Thanks!
Your while loop while True: works forever, so I guess your close or shutdown calls to the socket never can gets in way to work.
You should change while True: to while self.running: and that should do the trick.
Thanks ntki,rbp,st.
The problem solved use following code:
class treadListen(threading.Thread):
def __init__(self):
**self.running = True**
threading.Thread.__init__(self)
def run(self):
address = ('localhost', 16666)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
**sock.settimeout(1)**
sock.bind(address)
**while self.running:
try:
data = sock.recv(65535)
print "MESSAGE:{0}".format(data)
except Exception, e:
continue**
sock.close()
def stop(self):
**self.running = False**
# end of class thread_clock
I am trying to run a function in the background, whilst continuing with said code in python.
The function I want to run in the background is from socket. Looking for specific data to cut the program off.
Here is the function:
def receive():
host = ""
port = 13000
buf = 1024
addr = (host,port)
Sock = socket(AF_INET, SOCK_DGRAM)
Sock.bind(addr)
(data, addr) = Sock.recvfrom(buf)
return data
Here is the code I want to run:
while True:
r = receive()
if r == "stop":
break
#Cannot get past here, because of the function running.
#Should loop over and over, until stop data is received
print "Running program"
I have tried threading, with r = threading.Thread(target=receive()) with no joy.
Rookie error:
r = threading.Thread(target=receive())
I did not take the brackets off the receive():
r = threading.Thread(target=receive)
You can't return to the invoking thread from an invoked thread's target function. Instead, you need some inter-thread communication system. Below, is an example using Python's Queue to pass received datagrams between the two threads. I've used a threading.Event to signal when the receiver thread should stop.
#!/usr/bin/env python
import socket
import threading
from queue import Empty, Queue
class DatagramReceiver(threading.Thread):
def __init__(self, stop, queue):
super().__init__()
self._stop = stop
self._queue = queue
def run(self):
with socket.socket(AF_INET, SOCK_DGRAM) as sock:
sock.bind(('', 13000))
while not self._stop.is_set():
data = sock.recvfrom(1024)[0]
if data == 'stop':
self._stop.set()
break
self._queue.put(data)
def main():
stop = threading.Event()
queue = Queue()
reader = DatagramReceiver(stop, queue)
reader.deamon = True
reader.start()
while not stop.is_set():
user_input = input('Press RETURN to print datagrams, or q quit')
if user_input == 'q':
break
while True:
try:
datagram = queue.get_nowait()
except Empty:
break
print(datagram)
stop.set()
reader.join()
I'm new to rabbitmq and pika, and is having trouble with stopping consuming.
channel and queue setting:
connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()
channel.queue_declare(queue=new_task_id, durable=True, auto_delete=True)
Basically, consumer and producer are like this:
consumer:
def task(task_id):
def callback(channel, method, properties, body):
if body != "quit":
print(body)
else:
print(body)
channel.stop_consuming(task_id)
channel.basic_consume(callback, queue=task_id, no_ack=True)
channel.start_consuming()
print("finish")
return "finish"
producer:
proc = Popen(['app/sample.sh'], shell=True, stdout=PIPE)
while proc.returncode is None: # running
line = proc.stdout.readline()
if line:
channel.basic_publish(
exchange='',
routing_key=self.request.id,
body=line
)
else:
channel.basic_publish(
exchange='',
routing_key=self.request.id,
body="quit"
)
break
consumer task gave me output:
# ... output from sample.sh, as expected
quit
�}q(UstatusqUSUCCESSqU tracebackqNUresultqNUtask_idqU
1419350416qUchildrenq]u.
However, "finish" didn't get printed, so I'm guessing it's because channel.stop_consuming(task_id) didn't stop consuming. If so, what is the correct way to do? Thank you.
I had the same problem. It seems to be caused by the fact that internally, start_consuming calls self.connection.process_data_events(time_limit=None). This time_limit=None makes it hang.
I managed to workaround this problem by replacing the call to channel.start_consuming() with its implemenation, hacked:
while channel._consumer_infos:
channel.connection.process_data_events(time_limit=1) # 1 second
I have a class defined with member variables of channel and connection. These are initialized by a seperate thread. The consumer of MyClient Class uses the close() method and the the connection and consumer is stopped!
class MyClient:
def __init__(self, unique_client_code):
self.Channel = None
self.Conn: pika.BlockingConnection = None
self.ClientThread = self.init_client_driver()
def _close_callback(self):
self.Channel.stop_consuming()
self.Channel.close()
self.Conn.close()
def _client_driver_thread(self, tmout=None):
print("Starting Driver Thread...")
self.Conn = pika.BlockingConnection(pika.ConnectionParameters("localhost"))
self.Channel = self.Conn.channel()
def init_client_driver(self, tmout=None):
kwargs = {'tmout': tmout}
t = threading.Thread(target=self._client_driver_thread, kwargs=kwargs)
t.daemon = True
t.start()
return t
def close(self):
self.Conn.add_callback_threadsafe(self._close_callback)
self.ClientThread.join()
I'm running a python server using the socketserver module in python 2.7. OmniPeek packet analysis tool shows the TCP handshake completes,
but the server immediately sends a reset packet killing the connection.
Simplified server code which shows the problem is:
from threading import Lock, Thread, Condition
import SocketServer
import socket
import sys
import time
class ThreadedTCPRequestHandler(SocketServer.BaseRequestHandler):
def __init__(self, state, *args, **keys):
try:
state['lock'].acquire()
state['client_count'] += 1
finally:
state['lock'].release()
self.state = state
SocketServer.BaseRequestHandler.__init__(self, *args, **keys)
def handle(self):
self.state['lock'].acquire()
count = self.state['client_count']
self.state['lock'].release()
while True:
try:
self.state['lock'].acquire()
running = self.state['running']
self.state['lock'].release()
if not running:
break;
time.sleep(1) # do some work
except Exception as msg:
print msg
print "ThreadedTCPRequestHandler shutting down..."
class ThreadedTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
pass
def handler_factory(state):
def createHandler(*args, **keys):
return ThreadedTCPRequestHandler(state, *args, **keys)
return createHandler
if __name__ == "__main__":
lock = Lock()
cv = Condition(lock)
state = {'running': True, 'client_count': 0, 'lock': lock, 'cv': cv}
server = ThreadedTCPServer(('localhost', 12345), handler_factory(state))
server_thread = Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
print "Server loop running in thread:", server_thread.name
# wait for a client to connect
cv.acquire()
while state['client_count'] == 0 and state['running']:
cv.wait(1.0)
# print msg
cv.release()
# substitute real work here...
time.sleep(5)
lock.acquire()
state['running'] = False
lock.release()
server.shutdown()
and the client code:
import socket
if __name__ == "__main__":
try:
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print 'ip: {} port {}'.format('10.4.2.54', 12345)
client.connect(('10.4.2.54', 12345))
while True:
data = client.recv(4096)
if len(data) == 0:
break;
print 'data: {}'.format(data)
client.shutdown(socket.SHUT_RDWR)
client.close()
except Exception as msg:
print msg
The server code is based off python 2.7 docs serversocket Mixin example, and seems pretty straightforward, but...
Thanks
not sure what your expected behaviour is but if you make a couple of changes, you'll be able to see that it can work
replace your handle method
def handle(self):
while True:
try:
data = self.request.recv(1024).strip()
if len(data) != 0:
print data
time.sleep(1) # do some work
self.request.send('test data')
except Exception as msg:
print msg
break
print "ThreadedTCPRequestHandler shutting down..."
and client(inside main):
try:
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print 'ip: {} port {}'.format('localhost', 1232)
client.connect(('localhost', 1232))
client.send('test')
n = 0
while True:
data = client.recv(4096)
if len(data) != 0:
print 'data: {}'.format(data)
time.sleep(1)
n += 1
client.send('keep-alive' + str(n) + '\n')
print 'here'
client.shutdown(socket.SHUT_RDWR)
client.close()
except Exception as msg:
print msg
I just modded it to send stuff and print stuff. But it doesn't crash.
I think there is an issue with your self.state['lock'].acquire() and release() calls. I took out the 'running' check as it's not really used except at the end of the server code.
Also, without any action, sockets will time out.
Once again, I'm not claiming to have 'fixed' your problem...and I'm not sure exactly what you are looking for...just helping you brainstorm!
Apologies - red herring. The problem is only occurring under VM when server is running under guest and client is running under host. TCP reset packet never sent when both client and server are running either on host or guest