Develop Reference

Python - multithreaded sockets

From my understanding python can only run 1 thread at a time so if I were to do something like this
import socket, select
from threading import Thread
import config
class Source(Thread):
def __init__(self):
self._wait = False
self._host = (config.HOST, config.PORT + 1)
self._socket = socket.socket()
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._sock = None
self._connections = []
self._mount = "None"
self._writers = []
self._createServer()
Thread.__init__(self)
def _createServer(self):
self._socket.bind(self._host)
self._socket.listen(2)
self._connections.append(self._socket)
self._audioPackets=[]
def _addPacket(self, packet):
self._audioPackets.append(packet)
def _removePacket(self, packet):
self._audioPackets.remove(packet)
def _getPacket(self):
if len(self._audioPackets) > 0:
return self._audioPackets[0]
else:
return None
def _sendOK(self, sock):
sock.send("OK")
def _sendDenied(self, sock):
sock.send("DENIED")
def _sendMount(self, sock):
sock.send("mount:{0}".format(self._mount))
def _sendBufPacket(self, sock, packet):
packet = "buffer:%s" % packet
sock.send(packet)
def recv(self, sock, data):
data = data.split(":", 1)
if data[0] == "WAIT": self._wait = True
elif data[0] == "STOP_WAITING": self._wait = False
elif data[0] == "LOGIN":
if data[1] == config.SOURCE_AUTH:
self._source = sock
self._sendOK(sock)
else:
self._sendClose(sock)
elif data[0] == "MOUNT":
if self._source == sock:
self._mount = data[1]
else:
self._sendClose(sock)
elif data[0] == "CLIENT":
self._sendMount(sock)
self._writers.append(sock)
def _sendCloseAll(self):
for sock in self._connections:
sock.send("CLOSE")
sock.close()
def _sendClose(self, sock):
sock.send("CLOSE")
sock.close()
def main(self):
while True:
rl, wl, xl = select.select(self._connections, self._writers, [], 0.2)
for sock in rl:
if sock == self._socket:
con, ip = sock.accept()
self._connections.append(con)
else:
data = sock.recv(config.BUFFER)
if data:
self.recv(sock, data)
else:
if sock in self._writers:
self._writers.remove(sock)
if sock in self._connections:
self._connections.remove(sock)
for sock in wl:
packet = self._getPacket()
if packet != None:
self._sendBufPacket(sock, packet)
def run(self):
self.main()
class writeThread(Thread):
def __init__(self):
self.running = False
def make(self, client):
self.client = client
self.running = True
def run(self):
host = (config.HOST, config.PORT+1)
sock = socket.socket()
sock.connect(host)
sock.send("CLIENT")
sock.send("MOUNT:mountpoint")
while self.running:
data = sock.recv(config.BUFFER)
if data:
data = data.split(":", 1)
if data[0] == "buffer":
self.client.send(data[1])
elif data[0] == "CLOSE":
self.client.close()
break
if __name__=="__main__":
source = Source()
source.start()
webserver = WebServer()
webserver.runloop()
if I need to build the webserver part I will. But, I'll explain it.
Okay, so basically when someone connects to the websever under the mountpoint that was set, They will get there own personal thread that then grabs the data from Source() and sends it to them. Now say another person connects to the mount point and the last client as well as the source is still going. Wouldn't the new client be blocked from getting the Source data considering there are two active threads?

Your understanding of how Threads work in Python seems to be incorrect, based on the question you are asking. If used correctly, threads will not be blocking: you can instantiate multiple thread with Python. The limitation is that, due to the Global Interpreter Lock (GIL), you cannot get the full parallelism expected in thread programming (e.g. simultaneous execution and thus, reduced runtime).
What is going to happen in your case is that the two threads will take, together, the same amount of time that they would take if they were executed sequentially (although that is not necessarily what happens in practice).

Okay, I have copy and pasted some Python3 code that I have already written for a project that I am currently working on. With modification, you can make this code serve your purposes.
The code uses multiprocessing and multithreading. For my purposes, I am using multiprocessing so that sockets will run on one processor, and I can run a GUI program on another processor. You can remove the multiprocessor part if you prefer. The code below runs a socket message server. The server will listen for clients one at a time. Once a client has connected, a new thread will be initiated to handle all the communications between the server and each client. The server will then continue to search for for clients. At the moment however, the server only listens to data being sent from each client, and then it prints it to the terminal. With a small amount of effort, you can modify my code to send information from the server to each client individually.
import multiprocessing
import threading
from threading import Thread
class ThreadedServer(object):
def __init__(self, host, port):
self.host = host
self.port = port
self.sock = socket(AF_INET, SOCK_STREAM)
self.sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self.sock.bind((self.host, self.port))
def listen(self):
self.sock.listen(3) #Allow 3 Clients to connect to this server
while True:
#The program will search for one client at a time
print("Searching for Client")
client, address = self.sock.accept()
print(address, " is connected")
#client.settimeout(60)
#Once a client has been found, start a individual client thread
d = threading.Thread(target = self.listenToClient, args=(client, address))
d.daemon = True
d.start()
def listenToClient(self, client, address):
size = 1024
while True:
try:
data = client.recv(size)
if not data:
break
if data:
print(data)
#client.send(response)
else:
raise error('Client disconnected')
except:
client.close()
return False
def dataSharingHost():
#Using Sockets to send information between Processes
#This is the server Function
#ThreadServer(Host_IP, Port_Number), for LocalHost use ''
ThreadedServer('', 8000).listen()
def Main():
commServer = multiprocessing.Process(target=dataSharingHost, args=())
commServer.daemon = True
commServer.start()
if __name__== '__main__':
Main()
And to be fair, my code is modified from https://www.youtube.com/watch?v=qELZAi4yra8 . The client code is covered in those videos. I think the 3rd video covers the multiple client connects.

Python SocketServer.TCPServer errno 10054

I'm running a python server using the socketserver module in python 2.7. OmniPeek packet analysis tool shows the TCP handshake completes,
but the server immediately sends a reset packet killing the connection.
Simplified server code which shows the problem is:
from threading import Lock, Thread, Condition
import SocketServer
import socket
import sys
import time
class ThreadedTCPRequestHandler(SocketServer.BaseRequestHandler):
def __init__(self, state, *args, **keys):
try:
state['lock'].acquire()
state['client_count'] += 1
finally:
state['lock'].release()
self.state = state
SocketServer.BaseRequestHandler.__init__(self, *args, **keys)
def handle(self):
self.state['lock'].acquire()
count = self.state['client_count']
self.state['lock'].release()
while True:
try:
self.state['lock'].acquire()
running = self.state['running']
self.state['lock'].release()
if not running:
break;
time.sleep(1) # do some work
except Exception as msg:
print msg
print "ThreadedTCPRequestHandler shutting down..."
class ThreadedTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
pass
def handler_factory(state):
def createHandler(*args, **keys):
return ThreadedTCPRequestHandler(state, *args, **keys)
return createHandler
if __name__ == "__main__":
lock = Lock()
cv = Condition(lock)
state = {'running': True, 'client_count': 0, 'lock': lock, 'cv': cv}
server = ThreadedTCPServer(('localhost', 12345), handler_factory(state))
server_thread = Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
print "Server loop running in thread:", server_thread.name
# wait for a client to connect
cv.acquire()
while state['client_count'] == 0 and state['running']:
cv.wait(1.0)
# print msg
cv.release()
# substitute real work here...
time.sleep(5)
lock.acquire()
state['running'] = False
lock.release()
server.shutdown()
and the client code:
import socket
if __name__ == "__main__":
try:
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print 'ip: {} port {}'.format('10.4.2.54', 12345)
client.connect(('10.4.2.54', 12345))
while True:
data = client.recv(4096)
if len(data) == 0:
break;
print 'data: {}'.format(data)
client.shutdown(socket.SHUT_RDWR)
client.close()
except Exception as msg:
print msg
The server code is based off python 2.7 docs serversocket Mixin example, and seems pretty straightforward, but...
Thanks

not sure what your expected behaviour is but if you make a couple of changes, you'll be able to see that it can work
replace your handle method
def handle(self):
while True:
try:
data = self.request.recv(1024).strip()
if len(data) != 0:
print data
time.sleep(1) # do some work
self.request.send('test data')
except Exception as msg:
print msg
break
print "ThreadedTCPRequestHandler shutting down..."
and client(inside main):
try:
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print 'ip: {} port {}'.format('localhost', 1232)
client.connect(('localhost', 1232))
client.send('test')
n = 0
while True:
data = client.recv(4096)
if len(data) != 0:
print 'data: {}'.format(data)
time.sleep(1)
n += 1
client.send('keep-alive' + str(n) + '\n')
print 'here'
client.shutdown(socket.SHUT_RDWR)
client.close()
except Exception as msg:
print msg
I just modded it to send stuff and print stuff. But it doesn't crash.
I think there is an issue with your self.state['lock'].acquire() and release() calls. I took out the 'running' check as it's not really used except at the end of the server code.
Also, without any action, sockets will time out.
Once again, I'm not claiming to have 'fixed' your problem...and I'm not sure exactly what you are looking for...just helping you brainstorm!

Apologies - red herring. The problem is only occurring under VM when server is running under guest and client is running under host. TCP reset packet never sent when both client and server are running either on host or guest

Python - How to wait on both queue and a socket on same time

I want to read messages from either a Queue.Queue or a TCP socket, whichever comes first.
How can it be achieved without resorting to 2 threads ?
Platform is CPython 2.7.5 on Windows

There is a very nice trick to do this here that applies to your problem.
import queue
import socket
import os
class PollableQueue(queue.Queue):
def __init__(self):
super().__init__()
# Create a pair of connected sockets
if os.name == 'posix':
self._putsocket, self._getsocket = socket.socketpair()
else:
# Compatibility on non-POSIX systems
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('127.0.0.1', 0))
server.listen(1)
self._putsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._putsocket.connect(server.getsockname())
self._getsocket, _ = server.accept()
server.close()
def fileno(self):
return self._getsocket.fileno()
def put(self, item):
super().put(item)
self._putsocket.send(b'x')
def get(self):
self._getsocket.recv(1)
return super().get()

To do it in a single thread, you'll have to use non-blocking methods, and merge them into a single event loop. I'm actually using select instead of non-blocking socket I/O here, since it's slightly cleaner if you need to read from multiple sockets...
import socket
import select
import Queue
import time
TIMEOUT = 0.1 # 100ms
def process_queue_item(item):
print 'Got queue item: %r' % item
def process_socket_data(data):
print 'Got socket data: %r' % data
def main():
# Build queue
queue = Queue.Queue()
for i in range(10):
queue.put(i)
queue.put(None) # Using None to indicate no more data on queue
queue_active = True
# Build socket
sock = socket.socket()
sock.connect(('www.google.com', 80))
sock.send('GET / HTTP/1.0\r\n\r\n')
socket_active = True
# Main event loop
while 1:
# If there's nothing to read, bail out
if not (socket_active or queue_active):
break
# By default, sleep at the end of the loop
do_sleep = True
# Get data from socket without blocking if possible
if socket_active:
r, w, x = select.select([sock], [], [], TIMEOUT)
if r:
data = sock.recv(64)
if not data: # Hit EOF
socket_active = False
else:
do_sleep = False
process_socket_data(data)
# Get item from queue without blocking if possible
if queue_active:
try:
item = queue.get_nowait()
if item is None: # Hit end of queue
queue_active = False
else:
do_sleep = False
process_queue_item(item)
except Queue.Empty:
pass
# If we didn't get anything on this loop, sleep for a bit so we
# don't max out CPU time
if do_sleep:
time.sleep(TIMEOUT)
if __name__ == '__main__':
main()
Output looks like...
Got socket data: 'HTTP/1.0 302 Found\r\nLocation: http://www.google.co.uk/\r\nCache-Co'
Got queue item: 0
Got socket data: 'ntrol: private\r\nContent-Type: text/html; charset=UTF-8\r\nSet-Cook'
Got queue item: 1
Got socket data: 'ie: PREF=ID=a192ab09b4c13176:FF=0:TM=1373055330:LM=1373055330:S='
Got queue item: 2
etc.

You can do something along these lines:
def check_for_message(queue,socket,sock_accept_size=512):
socket.setblocking(0)
while True:
try:
sock_msg=socket.recv(sock_accept_size)
except socket.error:
"""Do stuff if there is no message"""
sock_msg=None
try:
que_msg=queue.get()
except Queue.Empty:
"""Do stuff if there is no message"""
que_msg=None
yield (que_msg,sock_msg)
Then you can iterate through it using:
for que_message,sock_message in check_for_message(que_instance,socket_instance):
print que_message,sock_message

Python socket.connect in for loop

I am writing a program in Python and a part of it is scanning for active ports on a website. in the module scanports, while if I were to say scan ports 79 to 81, I know that it should return a list with 80 in it. I know this for sure because when I run scanport it shows port 80 is up. Sorry for not having any comments:
import subprocess, socket, urllib2, sys
class pymap:
def __init__(self):
pass
################################################################################
################################################################################
def host(self, host):
self.host = host
socket1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock = socket1
################################################################################
################################################################################
def getip(self):
if self.host == None:
print "Specify a host first."
else:
return socket.gethostbyname(self.host)
################################################################################
################################################################################
def scanports(self, start, end):
ports = []
self.sock.settimeout(0.000001)
for i in xrange(start, end+1): #49151
try:
self.sock.connect((self.host, i))
ports.append(i)
except:
pass
return i
################################################################################
################################################################################
def scanport(self, port1):
self.sock.settimeout(0.000001)
try:
self.sock.connect((self.host, port1))
return 1
except:
return 0
################################################################################
################################################################################
def traceroute(self):
if self.host == None:
print "Specify a host first."
else:
proc=subprocess.Popen(('tracert', self.host), shell=True, stdout=subprocess.PIPE)
output=proc.communicate()[0]
return output
################################################################################
################################################################################
def getsource(self, url):
page = urllib2.urlopen(url)
return page.read()
################################################################################
################################################################################
x = pymap()
x.host("www.google.com")
print x.scanports(70, 85)
print x.scanport(80)
EDIT:
I changed it, thanks James Henstridge for pointing out that I was using the iteration variable, otherwise it would be much harder. However, it still doesn't work:
def scanports(self, start, end):
ports = []
self.sock.settimeout(3)
for i in xrange(start, end+1): #49151
try:
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect((self.host, port1))
self.sock.close()
ports.append(i)
except:
pass
return ports
EDIT: I figured it out, it was a problem with ports.append, thanks for your help.

There are a few issues with your code:
Your scanports method is returning the loop iteration variable, so it will always return end no matter what ports it detects. Perhaps you meant to return the ports list instead?
You are reusing the same socket over and over. From the connect system call man page:
Generally, connection-based protocol sockets may successfully connect() only once
So if you want to test multiple connection attempts, create a new socket each time. You should also close those sockets after use.

Python Access to BaseRequestHandler

My code basically needs to start up a simple chat server with a client. Where the server and the client can talk back and forth to each other. I've gotten everything to be implemented correctly, but I can't figure out how to shut down the server whenever I'm done. (I know it's ss.shutdown()).
I'm wanting to end right now based on a keyword shared between the two (something like "bye"), but I don't know if I can somehow send a message to my SocketServer from BaseRequestHandler to shutdown() whenever it receives the message.
Eventually, my goal is to incorporate Tkinter to make a GUI, but I wanted to get everything else to work first, and this is my first time dealing with sockets in Python.
from sys import argv, stderr
from threading import Thread
import socket
import SocketServer
import threading
import sys
class ThreadedRecv(Thread):
def __init__(self,socket):
Thread.__init__(self)
self.__socket = socket
self.__message = ''
self.__done = False
def recv(self):
while self.__message.strip() != "bye" and not self.getStatus():
self.__message = self.__socket.recv(4096)
print 'received',self.__message
self.setStatus(True)
def run(self):
self.recv()
def setStatus(self,status):
self.__done = status
def getStatus(self):
return self.__done
class ThreadedSend(Thread):
def __init__(self,socket):
Thread.__init__(self)
self.__socket = socket
self.__message = ''
self.__done = False
def send(self):
while self.__message != "bye" and not self.getStatus():
self.__message = raw_input()
self.__socket.send(self.__message)
self.setStatus(True)
def run(self):
self.send()
def setStatus(self,status):
self.__done = status
def getStatus(self):
return self.__done
class HostException(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
class EchoServer(SocketServer.BaseRequestHandler):
def setup(self):
print self.client_address, 'is connected!'
self.request.send('Hello ' + str(self.client_address) + '\n')
self.__done = False
def handle(self):
sender = ThreadedSend(self.request)
recver = ThreadedRecv(self.request)
sender.start()
recver.start()
while 1:
if recver.getStatus():
sender.setStatus(True)
break
if sender.getStatus():
recver.setStatus(True)
break
def finish(self):
print self.client_address, 'disconnected'
self.request.send('bye client %s\n' % str(self.client_address))
self.setDone(True)
def setDone(self,done):
self.__done = done
def getDone(self):
return self.__done
def setup(arg1, arg2, arg3):
server = False
defaultPort,defaultHost = 2358,"localhost"
hosts = []
port = defaultPort
serverNames = ["TRUE","SERVER","S","YES"]
arg1 = arg1.upper()
arg2 = arg2.upper()
arg3 = arg3.upper()
if arg1 in serverNames or arg2 in serverNames or arg3 in serverNames:
server = True
try:
port = int(arg1)
if arg2 != '':
hosts.append(arg2)
except ValueError:
if arg1 != '':
hosts.append(arg1)
try:
port = int(arg2)
if arg3 != '':
hosts.append(arg3)
except ValueError:
if arg2 != '':
hosts.append(arg2)
try:
port = int(arg3)
except ValueError:
if arg3 != '':
hosts.append(arg3)
port = defaultPort
for sn in serverNames:
if sn in hosts:
hosts.remove(sn)
try:
if len(hosts) != 1:
raise HostException("Either more than one or no host "+ \
"declared. Setting host to localhost.")
except HostException as error:
print error.value, "Setting hosts to default"
return (server,defaultHost,port)
return (server,hosts[0].lower(),port)
def main():
bufsize = 4096
while len(argv[1:4]) < 3:
argv.append('')
settings = setup(*argv[1:4])
connections = (settings[1],settings[2])
print connections
if not settings[0]:
try:
mySocket = socket.socket(socket.AF_INET,\
socket.SOCK_STREAM)
except socket.error, msg:
stderr.write("[ERROR] %s\n" % msg[1])
sys.exit(1)
try:
mySocket.connect(connections)
except socket.error, msg:
stderr.write("[ERROR] %s\n" % msg[1])
sys.exit(2)
message = ""
print "Enter a message to send to the server. "+\
"Enter \"bye\" to quit."
sender = ThreadedSend(mySocket)
recver = ThreadedRecv(mySocket)
sender.start()
recver.start()
while 1:
if sender.getStatus():
recver.setStatus(True)
break
if recver.getStatus():
sender.setStatus(True)
break
else:
xserverhandler = EchoServer
serversocket = SocketServer.ThreadedTCPServer(\
connections,xserverhandler)
server_thread = Thread(target = serversocket.serve_forever)
server_thread.setDaemon(True)
server_thread.start()
# I would like to shut down this server whenever
# I get done talking to it.
"""while 1:
if xserverhandler.getDone():
print 'This is now true!'
serversocket.shutdown()
break"""
if __name__ == '__main__':
main()
Yeah, I know setup() is a terrible function right now with the try's and catches, but it works for now, so I was going to fix it later.
My question is basically: How can I get the server to actually end based on a message that it receives? If possible, is there a way to access the Request Handler after it's started?

Please fix your code so it works, and include some way to use it. You need to add
class ThreadedTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
pass
since SocketServer doesn't actually include that class (at least not in my version of 2.6 nor 2.7). Instead, it's an example from the SocketServer definition.
Please include an example of how to start/use the code. In this case to start the server you need to do:
ss.py SERVER localhost 8001
and the client as
ss.py localhost 8001
If you do that then you can't do server_thread.setDaemon(True) because there are no other threads running, which means the server will exit immediately.
Once that's done the solution is to add a call (or two) to self.server.shutdown() insdie of your EchoServer.handle method, like:
while 1:
if recver.getStatus():
sender.setStatus(True)
self.server.shutdown()
break
However, I can't get that to work, and I think it's because I inherited things wrong, or guessed wrong in what you did.
What you should do is search for someone else who has done a chat server in Python. Using Google I found http://www.slideshare.net/didip/socket-programming-in-python and there are certainly others.
Also, if you are going to mix GUI and threaded programming then you should look into examples based on that. There are a number of hits when I searched for "tkinter chat". Also, you might want to look into twisted, which has solved a lot of these problems already.
What problems? Well, for example, you likely want an SO_REUSEADDR socket option.

Request handler object is created for each new request. So you have to store "done" flag in server, not handler. Something like the following:
class EchoServer(SocketServer.BaseRequestHandler):
...
def setDone(self):
self.server.setDone() # or even better directly self.server.shutdown()