Develop Reference

Passing a global variable though a multiprocess in Python

I have a sort of strange issue here. I am somewhat new to python and especially parallel TCP sockets in Python. I followed a guide to make a multi-client TCP server that runs using multiprocessing so I can maintain a front end control.
My understanding of passing variables in Python multiprocessing is using the args parameter when calling the multiprocessing.Process function.
However to start my TCP server my target isn't actually a def (function) and I am unsure how to properly pass a boolean value that will allow me to enable a debug mode in my code on the server process.
See code below:
import os
import datetime
import socket
import jetson.inference
import jetson.utils
import sys
import multiprocessing
import socketserver
from cv2 import *
class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler): #Class to process multiple clients if needed, handles sending and receiving messages as well as interperating them
def handle(self):
#cam_port = 0
if not os.path.exists("Images"):
os.makedirs("Images")
net = jetson.inference.detectNet("ssd-inception-v2", sys.argv, 0.5)
inputC = jetson.utils.videoSource("/dev/video0", argv=sys.argv)
# create video sources & outputs
#input = jetson.utils.videoSource(opt.input_URI, argv=sys.argv)
while True:
#print("Got here")
# self.request is the TCP socket connected to the client
self.data = self.request.recv(250).strip()
print("{} wrote:".format(self.client_address[0]))
rec = self.data.decode('utf-8')
print(rec)
if rec == "TRIGGER":
#cam = VideoCapture(cam_port)
#result, image = cam.read()
#processTrigger(cam, result, image, net)
processImage(net, inputC, debug)
double = "DOUBLE"
msg = double.encode('utf-8')
self.request.sendall(msg)
class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): #Class to bind each client to a thread
pass
def openServer(thread, server, isCreate): #function that handles opening and closing the tcp server
if isCreate:
with server:
print("Server running as ", thread.name, "on", server.server_address)
return thread
if not isCreate:
with server:
thread.terminate()
print("Server and Process terminated")
def processImage(network, inputC, debug):
# saving image in local storage
img = inputC.Capture()
detections = network.Detect(img, overlay="box,labels,conf")
print("detected {:d} objects in image".format(len(detections)))
print(debug)
for detection in detections:
print(detection)
if debug:
dateNow = datetime.datetime.now()
currentMSStr = dateNow.strftime("%f")
currentMS = round(int(currentMSStr)/1000)
outputFile = "Images/Capture_"+dateNow.strftime("%y_%m_%d_%H_%M_%S")+"_"+str(currentMS)+".png"
output = jetson.utils.videoOutput(outputFile, argv=sys.argv+['--headless'])
network.PrintProfilerTimes()
output.Render(img)
print("Warning! Debug mode is enabled, this should not be used during normal operation!")
debug = False #global debug boolean
if __name__ == "__main__":
enabled = True
serverOpen = False
HOST, PORT = "", 3334
print("Welcome to the Fives Vision Solution")
print("Press 1 to enable the server")
print("Press 2 to shutdown the server")
print("Press 3 to change server port")
print("Press 4 to enable debug mode. WARNING impacts performance!")
print("Press 5 to exit")
while enabled:
choice = input("Make a selection... \n")
if choice == '1': #Create socket and send to process to run in background
if serverOpen:
print("Server is already active")
if not serverOpen:
try:
server = ThreadedTCPServer((HOST, PORT), ThreadedTCPRequestHandler)
server_proc = multiprocessing.Process(target=server.serve_forever)
server_proc.daemon = True
server_proc.start()
openServer(server_proc,server,True)
serverOpen = True
except Exception as e:
print("Error occured: ", e)
elif choice == '2': #Terminate the server and close process
openServer(server_proc,server,False)
serverOpen = False
elif choice == '3': #Change port
if serverOpen:
print("Please close the server first.")
if not serverOpen:
print("Please enter a port:")
try:
port = input()
if int(port) > 1024 and int(port) < 65500:
PORT = int(port)
print("Port is now set to: ", PORT)
if int(port) < 1024 or int(port) > 65500:
print("Invalid port, must be a number between 1023 and 65500")
except:
print("Invalid port, must be a number between 1023 and 65500")
elif choice == '4': #Enable or disable debug mode
if debug:
debug = False
print("Debug set to: " ,debug)
continue
if not debug:
debug = True
print("Debug set to: " ,debug)
continue
elif choice == '5': #Terminate server is open and close the program
enabled = False
if (serverOpen):
openServer(server_proc,server,False)
else:
print("Invalid choice")
As you can see, line 87 initiates the server, and line 88 calls the actual server to serve in another process:
server = ThreadedTCPServer((HOST, PORT), ThreadedTCPRequestHandler)
server_proc = multiprocessing.Process(target=server.serve_forever)
How would I go about passing my global variable "debug" into my actual handle(self) function in my server class, which actually does the processing of the incoming message, calls a machine learning model and then returns the result back to the client.
Like I said, I'm pretty new to python so forgive me if there are better ways to do what I'm doing.
About the code:
My main issue is that I need debug mode because I want to be able to enable saving the images I process but doing so takes my total round trip of receiving a message, processing, and returning a result to around 200-250ms, and I need to run in less than 100ms.
When I disable the code within the debug if statement, my total receive/return time is around 40ms which is great, but if the system has issues I want to be able to store the results in image form for debugging camera issues and so on.
Thanks for any help!

Looking for help in making my socket messenger send instantaneously in Python

Im sure there are easier ways with particular python modules, but for an assignment I need to create a program that can act as a client/server. As of right now I have it working to the point of only being able to send a message if the reciever has responded. I need it to just send and appear on the respective client/server terminal when enter is pressed. Any help would be greatly appreciated!
These are pictures of what happens as of now
https://i.stack.imgur.com/T9CsJ.png
import sys
import socket
import getopt
def usage(script_name):
print('Usage: py' + script_name + '-l' +' <port number>' + '[<server>]')
def sockObj():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
return sock
def serversockConn(serversocket,port):
serversocket.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
# bind the serversocket to address
serversocket.bind(('',int(port)))
# enable a server to accept connections
serversocket.listen(5)
# wait for a connection and accept it
sock,addr = serversocket.accept()
return sock
def connToServersock(sock,server, port):
# connect to a serversocket
if server:
sock.connect((server, int(port)))
else:
sock.connect(('localhost', int(port)))
return sock
if __name__ == '__main__':
## get the command line arguments
try:
options, non_option_args = getopt.getopt(sys.argv[1:],'l')
except getopt.GetoptError as err:
print(err)
sys.exit(2)
#check if '-l' is present in command line agrument
serverSide = False
for opt,arg in options:
if opt == "-l":
serverSide = True
# port number
port = non_option_args[0]
# server address
server = None
hostLen = len(non_option_args)
if hostLen == 2:
server = non_option_args[1]
# create a communicator object, and make a connection between server and client
# server
if serverSide:
serversocket = sockObj()
sock = serversockConn(serversocket,port)
# client
else:
sock = sockObj()
sock = connToServersock(sock,server,port)
while True:
## read a message from standard input
message = sys.stdin.readline().encode()
if len(message) != 0:
sock.send(message)
return_msg = sock.recv( 1024 )
if return_msg:
print("Message recieved: " + return_msg.decode())
else:
print("Other side shut down")
else:
try:
sock.shutdown(socket.SHUT_WR)
sock.close()
except:
pass

I think your issue is that there are two places in your event loop where you block:
message = sys.stdin.readline().encode()
Here, you block until the user has pressed return -- during this time, your program is unable to respond to any data received over the network, because it is blocked waiting for data from stdin.
... and:
return_msg = sock.recv( 1024 )
Here, you are waiting for data to be received from the network -- during this time, your program is unable to respond to any data received from stdin, because it is blocked waiting for data from the network.
The behavior you'd ideally like to have is for your program to wait for both stdin and network traffic simultaneously -- i.e. have it block until either the user has pressed return, or some network data has been received, whichever comes first.
The easiest way to achieve that behavior is to use select(); its purpose is to block until at least one of several file descriptors is ready to be acted on. (Note, however, that Windows does not support using select() on stdin, so if your program needs to run under Windows you will probably have to spawn a second thread instead).
To implement the event loop using select(), add import select to the top of your script, then replace your event loop with something like this instead:
while True:
## block here until either sock or sys.stdin has data ready for us
readable, writable, exceptional = select.select([sock, sys.stdin], [], [])
if sys.stdin in readable:
## read a message from standard input
message = sys.stdin.readline().encode()
if len(message) != 0:
sock.send(message)
if sock in readable:
## read a message from the network
try:
return_msg = sock.recv( 1024 )
if (return_msg):
print("Message received: " + return_msg.decode())
else:
print("Other side shut down")
break
except:
print("recv() threw an exception")
break

Having trouble keeping socket alive Python

I'm trying to create a client/server console game, but I'm having trouble keeping the socket alive, it seems to be closing before I've called close() and I can't figure out why.
I've read the thread here but I have called connect() outside of the while loop since the logic of that already made sense to me before I attempted to run it, but I'm still getting the error from the server shell though:
ConnectionAbortedError: [WinError 10053] An established connection was aborted by the software in your host machine
So far I'm not getting an error on the client shell, it just exits running right after launching which is also when the server error occurs.
After fixing mistakes noted in the comments, the client now also shows an error stating that connected is undefined when attempting to start the while loop but should be since Connect() is run before entering the loop which should have set connected = True thus the loop should run but does not. I doubt this is related to the server issue at all, but what maybe the problem here?
I'll put the code for both below:
Client
import socket
def GetData():
data = s.recv(1000).decode()
if (data == "closing"):
connected = False
else:
print(data)
def SendData():
tdata = input("")
s.send(data.encode())
def Connect():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("127.0.0.1", 2222))
connected = True
global connected
Connect()
while (connected == True):
GetData()
if (connected == False):
s.close()
break
SendData()
Server
import socket
import random
def TheMainGame():
print ("started TheMainGame") # Remove after debug
def within(val, goal):
print ("Runing 'within' function") # Remove after debug
i = abs(val - goal)
if (i <= 5):
return True
else:
return False
def Guess():
print ("Running 'Guess' function") # Remove after debug
while True:
print ("Entered while loop") # Remove after debug
data = "What is your guess?"
c.send(data.encode())
print ("Sent data") # Remove after debug
t = c.recv(1000).decode()
print ("Got data") # Remove after debug
if (t == x):
data = "Correct!"
c.send(data.encode())
data = "closing"
c.send(data.encode())
c.close()
break
elif (within(t,x) == True):
data = "You're close!"
c.send(data.encode())
elif (within(t,x) == False):
data = "Try again"
c.send(data.encode())
else:
data = "There was an error computing the value"
c.send(data.encode())
c.close()
break
x = random.randrange(1, 20, 1)
Guess()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("127.0.0.1", 2222))
s.listen(5)
global data
while True:
(c,a) = s.accept()
print("Received connection from", a) # Remove after debug
data = "Hello %s" %a[0] # Remove after debug
c.send(data.encode()) # Remove after debug
TheMainGame()
The error in the server shell relates to line 19 which is why the server attempts to ask the client the question, which is also the second time it attempts to send data to the client but the attempt never successfully happens, which is why I think the socket is being closed even though I never told it to anywhere before that point since it hasn't even been able to check if t == x. What is causing the socket to close?

You're mis-using global in the client.
in the main body you have global connected. This is incorrect.
Instead, at the top of the file put:
import socket
connected = False
Then, in the Connect function, pull in the global scoped variable:
def Connect():
global connected
...
The keyword global is to tell a function to use a variable from the global scope, not to define them.
Note:
There are other errors in this code which you will need to look at and address.
You've done the same in server.py
You're trying to reference socket s in the GetData and SendData functions but s is local to within the connect function only (meaning that the connection will drop once the function exists.

Using Pygame with python sockets, black screen

I have been trying to create a two player game in pygame. I did some research on sockets and have been trying to put them into the game. This is my setup sockets function in the server, I won't put all the game code in since its quite long:
def create_sockets(self):
self.ip = "192.168.1.68"
self.port = 8888
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.ip, self.port))
self.server_socket.listen(5)
self.conn, self.addr = self.server_socket.accept()
But through trial and error, when server_socket.accept() is run, I get a black screen and a color wheel (I'm on a mac). Why is this happening? the same code works fine in my server test from before. Since I am very new to sockets please correct me on any mistakes/bad practice
Thanks in advance

+1 to svk. Whenever I use Socket in pygame, I use threading to make it asynchronous. I'm pretty sure both .listen() and .accept() will freeze your program in a loop as those methods are waiting for something to happen.
Here is the full code for a pong clone in pygame. This is a "dumb" server, meaning it is not handling game logic, just sharing data with clients.
This may not be your preferred approach as a whole, but it does show how to handle asynchronous connection/listening. As well as using pickle to encode/decode whatever data type you want. That way you can kick lists around and stuff.
# Server example:
from threading import Thread
import socket, pickle, logging
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("0.0.0.0", 12354))
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(lineno)s %(message)s', level=logging.DEBUG)
client_list = []
max_clients = 2
global started
started = 0
class Client():
def __init__(self, conn = ''):
self.conn = conn
# add to global clients list
client_list.append(self)
self.client_thread = Thread(target = self.process_messages)
self.client_thread.start()
def process_messages(self):
while True:
try:
data = self.conn.recv(1024)
# send to all in client_list except self
data = pickle.loads(data)
data.append(started)
logging.info("Sending Data: {0}".format(data))
data = pickle.dumps(data)
for client in client_list:
if client != self:
client.conn.sendall(data)
data = ""
except ConnectionResetError:
logging.debug("Client Disconnected")
break
def connection_manager():
while len(client_list) < max_clients:
logging.info('Listening for connections...')
s.listen(1)
conn, addr = s.accept()
logging.info("Client connected: {0}".format(addr))
x = Client(conn)
logging.debug(client_list)
logging.warning("Max clients reached")
logging.info("No longer listening..")
started = 0
accept_connections_thread = Thread(target = connection_manager)
accept_connections_thread.start()

My chat client freezes up after beginning threads

I made a better chat client following help from people:
They told me that if I didn't want to be blocked on .recv when waiting for messages, I would need to use threads, classes, functions, and queues to do so.
So I followed some help a specific person gave me where I created a thread from a class and then defined a function that was supposed to read incoming messages and print them.
I also created a function that allows you to enter stuff to be sent off.
Thing is, when I run the program. Nothing happens.
Can somebody help point out what is wrong? (I've asked questions and researched for 3 days, without getting anywhere, so I did try)
from socket import *
import threading
import json
import select
print("Client Version 3")
HOST = input("Connect to: ")
PORT = int(input("On port: "))
# Create Socket
s = socket(AF_INET,SOCK_STREAM)
s.connect((HOST,PORT))
print("Connected to: ",HOST,)
#-------------------Need 2 threads for handling incoming and outgoing messages--
# 1: Create out_buffer:
Buffer = []
rlist,wlist,xlist = select.select([s],Buffer,[])
class Incoming(threading.Thread):
# made a function a thread
def Incoming_messages():
while True:
for i in rlist:
data = i.recv(1024)
if data:
print(data.decode())
# Now for outgoing data.
def Outgoing():
while True:
user_input=("Your message: ")
if user_input is True:
Buffer += [user_input.encode()]
for i in wlist:
s.sendall(Buffer)
Buffer = []
Thanks for taking a look, thanks also to Tony The Lion for suggesting this

Take a look at this revised version of your code: (in python3.3)
from socket import *
import threading
import json
import select
print("client")
HOST = input("connect to: ")
PORT = int(input("on port: "))
# create the socket
s = socket(AF_INET, SOCK_STREAM)
s.connect((HOST, PORT))
print("connected to:", HOST)
#------------------- need 2 threads for handling incoming and outgoing messages--
# 1: create out_buffer:
out_buffer = []
# for incoming data
def incoming():
rlist,wlist,xlist = select.select([s], out_buffer, [])
while 1:
for i in rlist:
data = i.recv(1024)
if data:
print("\nreceived:", data.decode())
# now for outgoing data
def outgoing():
global out_buffer
while 1:
user_input=input("your message: ")+"\n"
if user_input:
out_buffer += [user_input.encode()]
# for i in wlist:
s.send(out_buffer[0])
out_buffer = []
thread_in = threading.Thread(target=incoming, args=())
thread_out = threading.Thread(target=outgoing, args=())
thread_in.start() # this causes the thread to run
thread_out.start()
thread_in.join() # this waits until the thread has completed
thread_out.join()
in your program you had various problems, namely you need to call the threads; to just define them isn't enough.
you also had forgot the function input() in the line: user_input=input("your message: ")+"\n".
the "select()" function was blocking until you had something to read, so the program didn't arrive to the next sections of the code, so it's better to move it to the reading thread.
the send function in python doesn't accept a list; in python 3.3 it accepts a group of bytes, as returned by the encoded() function, so that part of the code had to be adapted.