Develop Reference

Why can't I access a specific variable inside of a threaded class

The bounty expires in 5 days. Answers to this question are eligible for a +50 reputation bounty.
Haley Mueller wants to draw more attention to this question.
I'm new to Python so this could be a simple fix.
I am using Flask and sockets for this Python project. I am starting the socket on another thread so I can actively listen for new messages. I have an array variable called 'SocketConnections' that is within my UdpComms class. The variable gets a new 'Connection' appended to it when a new socket connection is made. That works correctly. My issue is that when I try to read 'SocketConnections' from outside of the thread looking in, it is an empty array.
server.py
from flask import Flask, jsonify
import UdpComms as U
app = Flask(__name__)
#app.route('/api/talk', methods=['POST'])
def talk():
global global_server_socket
apples = global_server_socket.SocketConnections
return jsonify(message=apples)
global_server_socket = None
def start_server():
global global_server_socket
sock = U.UdpComms(udpIP="127.0.0.1", portTX=8000, portRX=8001, enableRX=True, suppressWarnings=True)
i = 0
global_server_socket = sock
while True:
i += 1
data = sock.ReadReceivedData() # read data
if data != None: # if NEW data has been received since last ReadReceivedData function call
print(data) # print new received data
time.sleep(1)
if __name__ == '__main__':
server_thread = threading.Thread(target=start_server)
server_thread.start()
app.run(debug=True,host='192.168.0.25')
UdpComms.py
import json
import uuid
class UdpComms():
def __init__(self,udpIP,portTX,portRX,enableRX=False,suppressWarnings=True):
self.SocketConnections = []
import socket
self.udpIP = udpIP
self.udpSendPort = portTX
self.udpRcvPort = portRX
self.enableRX = enableRX
self.suppressWarnings = suppressWarnings # when true warnings are suppressed
self.isDataReceived = False
self.dataRX = None
# Connect via UDP
self.udpSock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # internet protocol, udp (DGRAM) socket
self.udpSock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # allows the address/port to be reused immediately instead of it being stuck in the TIME_WAIT state waiting for late packets to arrive.
self.udpSock.bind((udpIP, portRX))
# Create Receiving thread if required
if enableRX:
import threading
self.rxThread = threading.Thread(target=self.ReadUdpThreadFunc, daemon=True)
self.rxThread.start()
def __del__(self):
self.CloseSocket()
def CloseSocket(self):
# Function to close socket
self.udpSock.close()
def SendData(self, strToSend):
# Use this function to send string to C#
self.udpSock.sendto(bytes(strToSend,'utf-8'), (self.udpIP, self.udpSendPort))
def SendDataAddress(self, strToSend, guid):
# Use this function to send string to C#
print('finding connection: ' + guid)
if self.SocketConnections:
connection = self.GetConnectionByGUID(guid)
print('found connection: ' + guid)
if connection is not None:
self.udpSock.sendto(bytes(strToSend,'utf-8'), connection.Address)
def ReceiveData(self):
if not self.enableRX: # if RX is not enabled, raise error
raise ValueError("Attempting to receive data without enabling this setting. Ensure this is enabled from the constructor")
data = None
try:
data, _ = self.udpSock.recvfrom(1024)
print('Socket data recieved from: ', _)
if self.IsNewConnection(_) == True:
print('New socket')
self.SendDataAddress("INIT:" + self.SocketConnections[-1].GUID, self.SocketConnections[-1].GUID)
data = data.decode('utf-8')
except WindowsError as e:
if e.winerror == 10054: # An error occurs if you try to receive before connecting to other application
if not self.suppressWarnings:
print("Are You connected to the other application? Connect to it!")
else:
pass
else:
raise ValueError("Unexpected Error. Are you sure that the received data can be converted to a string")
return data
def ReadUdpThreadFunc(self): # Should be called from thread
self.isDataReceived = False # Initially nothing received
while True:
data = self.ReceiveData() # Blocks (in thread) until data is returned (OR MAYBE UNTIL SOME TIMEOUT AS WELL)
self.dataRX = data # Populate AFTER new data is received
self.isDataReceived = True
# When it reaches here, data received is available
def ReadReceivedData(self):
data = None
if self.isDataReceived: # if data has been received
self.isDataReceived = False
data = self.dataRX
self.dataRX = None # Empty receive buffer
if data != None and data.startswith('DIALOG:'): #send it info
split = data.split(':')[1]
return data
class Connection:
def __init__(self, gUID, address) -> None:
self.GUID = gUID
self.Address = address
def IsNewConnection(self, address):
for connection in self.SocketConnections:
if connection.Address == address:
return False
print('Appending new connection...')
connection = self.Connection(str(uuid.uuid4()),address)
self.SocketConnections.append(connection)
return True
def GetConnectionByGUID(self, guid):
for connection in self.SocketConnections:
if connection.GUID == guid:
return connection
return None
As mentioned above. When IsNewConnection() is called in UdpComms it does append a new object to SocketConnections. It's just trying to view the SocketConnections in the app.route that is empty. My plans are to be able to send socket messages from the app.routes

For interprocess communication you may try to use something like shared memory documented here
Instead of declaring your self.SocketConnections as a list = []
you'd use self.SocketConnections = Array('i', range(10)) (you are then limited to remembering only 10 connections though).

Python Socket with Multiprocessing and Pickle issue

I am having a Pickle issue with SSL client to server communication using multiprocessing.
I have an SSL client that connects to the server:
SSLClient.py
import socket
import struct
import ssl
import copyreg
from os import path
import socket
import os
from pathlib import Path
from loguru import logger as log
from utils.misc import read_py_config
from datetime import datetime
from cryptography.fernet import Fernet
fernetkey = '1234567'
fernet = Fernet(fernetkey)
class SSLclient:
license = None
licenseencrypted = None
uuid = None
def __init__(self):
try:
path = Path(__file__).parent / "/lcl" #get unique license key
with path.open() as file:
self.licenseencrypted = file.read().rstrip()
self.license = fernet.decrypt(str.encode(self.licenseencrypted)).decode('ascii')
self.host, self.port = "127.0.0.1", 65416
except Exception as e:
log.error("Could not decode license key")
def connect(self):
self.client_crt = os.path.join(os.path.dirname(__file__), 'key/c-.crt')
self.client_key = os.path.join(os.path.dirname(__file__), 'key/ck-.key')
self.server_crt = os.path.join(os.path.dirname(__file__), 'key/s-.crt')
self.sni_hostname = "example.com"
self._context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=self.server_crt)
self._context.load_cert_chain(certfile=self.client_crt, keyfile=self.client_key)
self._sock = None
self._ssock = None
## ---- Client Communication Setup ----
HOST = self.host # The server's hostname or IP address
PORT = self.port # The port used by the server
try:
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._ssock = self._context.wrap_socket(self._sock, server_side=False, server_hostname=self.sni_hostname)
self._ssock.connect((HOST, PORT))
log.info("Socket successfully created")
except socket.error as err:
log.error("socket creation failed with error %s" %(err))
return False
log.info('Waiting for connection')
return True
def closesockconnection(self):
self._ssock.close()
def checkvalidsite(self):
#check if site is active
jsonobj = {
"uuid": self.license,
"ipaddress" : self.external_ip,
"req": "checkvalidsite"
}
send_msg(self._ssock, json.dumps(jsonobj).encode('utf-8'))
active = False
while True:
Response = recv_msg(self._ssock)
if not Response:
return False
if Response is not None:
Response = Response.decode('utf-8')
Response = json.loads(Response)
req = Response['req']
if req == "checkvalidsite":
active = Response['active']
self.info1 = Response['info1']
self.info2 = Response['info2']
return active
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
try:
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
except Exception as e:
log.error("Sending message " + str(e))
def recv_msg(sock: socket): # ---- Use this to receive
try:
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
except Exception as e:
log.error("Receiving message " + str(e))
return False
def recvall(sock: socket, n: int):
try:
# Helper function to receive n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
except Exception as e:
log.error("Receiving all message " + str(e))
raise Exception(e)
I then have a server that is Multithreaded and accepts the connection and communicates with the client.
Server.py
import socket
import os
from socket import AF_INET, SOCK_STREAM, SO_REUSEADDR, SOL_SOCKET, SHUT_RDWR
import ssl
from os import path
from _thread import *
import struct # Here to convert Python data types into byte streams (in string) and back
import traceback
from threading import Thread
import json
import mysql.connector as mysql
import time
from loguru import logger as log
import threading
from cryptography.fernet import Fernet
fernetkey = '12213423423'
fernet = Fernet(fernetkey)
threadLocal = threading.local()
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
try:
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
except Exception as e:
log.error("Error send_msg " + str(e))
def recv_msg(sock: socket): # ---- Use this to receive
try:
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
except Exception as e:
log.error("Receiving message " + str(e))
return False
def recvall(sock: socket, n: int):
try:
# Helper function to receive n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
except Exception as e:
log.error("Receiving all message " + str(e))
raise Exception(e)
# ---- Server Communication Setup
class Newclient:
def __init__(self):
self.addr = None
self.conn = None
self.uuid = None
class Server:
def __init__(self):
self.HOST = '127.0.0.1' # Standard loopback interface address (localhost)
self.PORT = 65416 # Port to listen on (non-privileged ports are > 1023)
self.ThreadCount = 0
self.threads = []
self.sock = None
def checkvalidsite(self, uuid, ipaddress, cursor, db_connection):
sql = "select * from myexample where uuid ='" + uuid + "'"
cursor.execute(sql)
results = cursor.fetchall()
active = False
for row in results:
active = row["active"]
siteid = row["info1"]
clientid = row["info2"]
return active, siteid, clientid
def Serverthreaded_client(self, newclient):
conn = newclient.conn
try:
while True:
# data = conn.recv(2048) # receive message from client
data = recv_msg(conn)
uuid = None
ipaddress = None
req = None
if not data :
return False
if data is not None:
data = json.loads(data.decode('utf-8'))
uuid = data['uuid']
req = data['req']
if uuid is not None and req is not None:
newclient.uuid = uuid
cursor, db_connection = setupDBConnection()
if req == "checkvalidsite":
ipaddress = data['ipaddress']
active, info1, info2 = self.checkvalidsite(uuid, ipaddress, cursor, db_connection)
data = {
"req": "checkvalidsite",
"uuid": uuid,
"active": active,
"info1" : info1,
"info2" : info2
}
if not data:
break
# conn.sendall(str.encode(reply))
send_msg(conn, json.dumps(data).encode('utf-8'))
log.info("Server response sent")
#conn.close()
closeDBConnection(cursor, db_connection)
else:
#send no message
a=1
except Exception as e:
log.warning(str(e))
log.warning(traceback.format_exc())
finally:
log.info("UUID Closing connection")
conn.shutdown(socket.SHUT_RDWR)
conn.close()
#conn.close()
def Serverconnect(self):
try: # create socket
self.server_cert = path.join(path.dirname(__file__), "keys/server.crt")
self.server_key = path.join(path.dirname(__file__), "keys/server.key")
self.client_cert = path.join(path.dirname(__file__), "keys/client.crt")
self._context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
self._context.verify_mode = ssl.CERT_REQUIRED
###self._context.load_cert_chain(self.server_cert, self.server_key)
self._context.load_cert_chain(certfile=self.server_cert, keyfile=self.server_key)
###self._context.load_verify_locations(self.client_cert)
self._context.load_verify_locations(cafile=self.client_cert)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) ###<-- socket.socket() ???
log.info("Socket successfully created")
except socket.error as err:
log.warning("socket creation failed with error %s" %(err))
try: # bind socket to an address
self.sock.bind((self.HOST, self.PORT))
except socket.error as e:
log.warning(str(e))
log.info('Waiting for a Connection..')
self.sock.listen(3)
def Serverwaitforconnection(self):
while True:
Client, addr = self.sock.accept()
conn = self._context.wrap_socket(Client, server_side=True)
log.info('Connected to: ' + addr[0] + ':' + str(addr[1]))
log.info("SSL established. Peer: {}".format(conn.getpeercert()))
newclient = Newclient()
newclient.addr = addr
newclient.conn = conn
thread = Thread(target=self.Serverthreaded_client, args =(newclient, ))
thread.start()
self.threads.append(newclient)
self.ThreadCount += 1
log.info('Thread Number: ' + str(self.ThreadCount))
def startserver():
server = Server()
server.Serverconnect()
server.Serverwaitforconnection()
serverthread = Thread(target=startserver)
serverthread.daemon = False
serverthread.start()
The server accepts the connection with SSL then waits for a message. It investigates the message command, executes the respective function and returns the data from the database as a response (checkvalidsite in this example).
All good so far (as far as I can tell).
I also have the main program that calls the SSLClient and connects.
Main program
remoteclient = SSLclient()
successfulconnection = remoteclient.connect()
siteactive = remoteclient.checkvalidsite()
So far all is well. However I also have the main program reading in frames from multiple cameras. Can be 20 cameras for example. In order to do this I created multiprocessing to deal with the camera load. Each camera or two cameras per, are assigned to a processor (depending on the number of cores in the machine).
(code below has been stripped out to simplify reading)
x = range(3, 6)
for n in x:
processes = multiprocessing.Process(target=activateMainProgram, args=(queue1, queue2, queue3, queue4, remoteclient, ))
processes.start()
When I try pass the remoteclient (SSLClient) as an argument I get the error:
cannot pickle 'SSLContext' object
I then (after reading online) added the code to the SSLClient:
def save_sslcontext(obj):
return obj.__class__, (obj.protocol,)
copyreg.pickle(ssl.SSLContext, save_sslcontext)
but then I get the error:
cannot pickle 'SSLContext' object
There are 2 options I experimented with:
Trying to get the pickle working (which would be ideal) as the processes themselves each need to communicate with the server. So the processes need to call functions from the SSLClient file. But I cannot get over the pickle issue and can't find a solution online
I then placed the remoteclient = SSLClient code outside the main function. Hoping it would run first and then be accessible to the processes. This worked, however what I learnt was that when a process is called (as it does not share memory) it reprocesses the entire file. Meaning if I have 10 processes each with 2 cameras then I would have 10 connections to the server (1 per process). This means on the server side I would also have 10 threads running each connection. Though it works, it seems significantly inefficient.
Being a noob and self taught in Python I am not sure how to resolve the issue and after 3 days, I figured I would reach out for assistance. If I could get assistance with the pickle issue of the SSLClient then I will have one connection that is shared with all processes and 1 thread in the server to deal with them.
P.s. I have cobbled all of the code together myself and being new to Python if you see that I am totally going down the wrong, incorrect, non-professional track, feel free to yell.
Much appreciated.
Update:
If I change the SSLClient code to:
def save_sslcontext(obj):
return obj.__class__, (obj.protocol,)
copyreg.pickle(ssl.SSLContext, save_sslcontext)
Then I get the error:
[WinError 10038] An operation was attempted on something that is not a socket
Not sure what is better..

Sending and receiving package in python

This is a console chat app on a TCP socket server. The client will send the request/message to the server and the server will distribute the message to the target user or provide requested information.I am currently running into a problem regarding the recv package on the server side. I received the package and was able to print it out. However the system still give me a syntax error for some reason.
Thanks.
This is my client:
import socket
import select
import errno
import sys, struct
import pickle
HEADER_LENGTH = 1024
IP = "127.0.0.1"
PORT = 9669
def send_login_request(username):
package = [1]
length = len(username)
if length > 1019:
print ("Error: Username too long")
sys.exit()
package += struct.pack("I", length)
package += username
return package
def send_message(recv_id, message):
package = [2]
length = len(message)
if length > 1015:
print('message too long')
sys.exit()
package += recv_id
package += struct.pack('I', length)
package += message
return package
def send_con_request(conv_id):
package = [3]
length = len(id)
if length > 1015:
print('id too long')
sys.exit()
package += struct.pack("I", length)
package += conv_id
return package
# Create a socket
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Connect to a given ip and port
client_socket.connect((IP, PORT))
client_socket.setblocking(False)
my_username = input("Username: ")
request = send_login_request(my_username)
user_request = str(request)
client_socket.send(user_request.encode())
username_conf = client_socket.recv(HEADER_LENGTH).decode()
if username_conf == "Welcome to the server":
con_id = input("Please enter conversation's id, if don't have one, please enter no ")
if con_id == 'no':
con_request = send_con_request(con_id)
con_request = str(con_request)
client_socket.send(con_request.encode())
else:
con_request = send_con_request(con_id)
con_request = str(con_request)
client_socket.send(con_request.encode())
conversation = client_socket.recv(HEADER_LENGTH).decode()
recv_id = input("Please enter receiver's id")
while True:
# Wait for user to input a message
message = input(f'{my_username} > ').encode()
# If message is not empty - send it
if message:
send_message = send_message(recv_id,message)
client_socket.send(bytes(send_message))
try:
while True:
message_receiver = client_socket.recv(HEADER_LENGTH).decode()
x = message_receiver.split('|')
print(x)
username = x[0]
message = x[1]
# Print message
print(f'{username} > {message}')
except IOError as e:
if e.errno != errno.EAGAIN and e.errno != errno.EWOULDBLOCK:
print('Reading error: {}'.format(str(e)))
sys.exit()
# We just did not receive anything
continue
except Exception as e:
# Any other exception - something happened, exit
print('Reading error: {}'.format(str(e)))
sys.exit()
This is my server:
import socket
import select
import struct
import sys
import pickle
HEADER_LENGTH = 1024
conversation ={}
users = [
{
'username': 'user1',
'user_id': 1
},
{
'username': 'user2',
'user_id': 2
},
{
'username': 'user3',
'user_id': 3
},
{
'username': 'user4',
'user_id': 4
},
{
'username': 'user5',
'user_id': 5
}
]
def login(username):
for user in users:
if user['username'] == username:
return user
else:
return False
IP = "127.0.0.1"
PORT = 9669
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((IP, PORT))
server_socket.listen()
# List of sockets for select.select()
sockets_list = [server_socket]
# List of connected clients - socket as a key, user header and name as data
clients_socket = {}
sessions = {
(1,2) : '1.txt',
(3,4) : '2.txt'
}
def getRecvSocket(user_id):
try:
return sessions[user_id]
except:
return None
def sendErrorMes(socketid, mes):
package = [9]
length = len(mes)
if length > 1019:
length = 1019
package += struct.pack("I", length)
package += mes
print(f'Listening for connections on {IP}:{PORT}...')
# Handles message receiving
def receive_message(client_socket):
try:
receive_message = client_socket.recv(HEADER_LENGTH)
return receive_message
except:
return False
while True:
read_sockets, _, exception_sockets = select.select(sockets_list, [], sockets_list)
# Iterate over notified sockets
for notified_socket in read_sockets:
# If notified socket is a server socket - new connection, accept it
if notified_socket == server_socket:
client_socket, client_address = server_socket.accept()
sockets_list.append(client_socket)
else:
# Receive message
package = receive_message(notified_socket)
print(package)
package_recv = eval(package.decode())
print(package_recv)
print(type(package_recv))
package_type = package_recv[0]
if package_type == 1:
size = struct.unpack("I", package[1:5])
if size[0] > 1019:
continue
username = package[5:5+size[0]]
username = username.decode()
# username = package_recv[1]
user = login(username)
if user == False:
notified_socket.send("no user found".encode())
else:
sessions[user["user_id"]] = notified_socket
notified_socket.send(("Welcome to the server").encode())
elif package_type == 2:
recv_id = struct.unpack("I", package[1:5])
size = struct.unpack("I", package[5:9])
if size[0] > 1015:
continue
# recv_id = package_recv[1]
if getRecvSocket(recv_id) == None:
sendErrorMes(notified_socket, "User is offline")
else:
message = package[9:9+size[0]]
# message = package_recv[2]
for socket in sessions.values():
if socket == notified_socket:
user = sessions[notified_socket]
# print(f'Received message from {user}, {message}')
# fIterate over connected clients and broadcast message
for client_socket in clients_socket:
# if clients[client_socket] == receive_user and client_socket != notified_socket:
# But don't sent it to sender
if client_socket != notified_socket and clients_socket[client_socket] == recv_id:
# Send user and message (both with their headers)
# We are reusing here message header sent by sender, and saved username header send by user when he connected
a = sessions[notified_socket]
b = recv_id
with open(f"{conversation[a,b]}.txt", "w"):
f.write(user + message)
client_socket.send((user + "|" + message).encode())
if message is False:
# print('Closed connection from: {}'.format(user))
# Remove from list for socket.socket()
sockets_list.remove(notified_socket)
# Remove from our list of users
del clients_socket[notified_socket]
continue
elif package_type == 3:
size = struct.unpack("I", package[1:5])
if size[0] > 1019:
continue
convo_id = package[5:5+size[0]]
convo_id = convo_id.decode()
# convo_id = package_recv[2]
if convo_id in conversation:
with open(conversation[convo_id], 'rb') as file_to_send:
for data in file_to_send:
notified_socket.sendall(data)
print('send successful')
else:
f = open(f"{len(conversation)+1}.txt", "w+")
This is the error in the server side which I am having a problem to locate and solve:
Listening for connections on 127.0.0.1:9669...
b"[1, 5, 0, 0, 0, 'u', 's', 'e', 'r', '1']"
[1, 5, 0, 0, 0, 'u', 's', 'e', 'r', '1']
<class 'list'>
b''
Traceback (most recent call last):
File "c:/Users/Duong Dang/Desktop/bai 2.3/server.py", line 134, in <module>
package_recv = eval(package.decode())
File "<string>", line 0
^
SyntaxError: unexpected EOF while parsing

The sending code doesn't make a lot of sense. You're creating a python list which is a most strange way to implement a protocol. You're then taking python's string representation of that list and sending it to the server. You're not doing anything on the server side to ensure that you got the entire message. Then you're using eval to interpret the string you created on the client. That is a very dangerous practice, as your peer can essentially instruct your python interpreter to do literally anything.
Also, your send_con_request is calling len(id) which won't work at all because id is a python built-in that doesn't supply a __len__ method. I assume that was supposed to be len(conv_id)?
Anyway, you should rework your protocol. Use the struct tools to create the correct binary string you want. There are tons of possible ways to structure this but here's one. On the client side, create a fixed-length header that identifies which request type you're sending and the length of the remaining "payload" bytes. You'll convert your string (username or whatever) into bytes first with str.encode.
import struct
# ProtoHeader encodes a 16 bit request identifer, plus a 32 bit payload
# length. A protocol data unit consists of this 6-byte header followed by
# payload bytes (which will vary according to the request)
ProtoHeader = struct.Struct("!HI")
LoginRequest = 1
SomeOtherRequest = 2
...
def format_login_request(username):
""" Create a protocol block containing a user login request.
Return the byte string containing the encoded request """
username_bytes = username.encode()
proto_block = ProtoHeader.pack(LoginRequest, len(username_bytes)) + username_bytes
return proto_block
...
conn.sendall(format_login_request(username))
On the server side, you will first receive the fixed-length header (which tells you what the request type was and how many other payload bytes are present). Then receive those remaining bytes ensuring that you get exactly that many. socket.recv does not guarantee that you will receive exactly the number of bytes sent in any particular send from the peer. It does guarantee that you will get them in the right order so you must keep receiving until you got exactly the number you expected. That's why it's important to have fixed length byte strings as a header and to encode the number of bytes expected in variable length payloads.
The server would look something like this:
import struct
ProtoHeader = struct.Struct("!HI")
LoginRequest = 1
def receive_bytes(conn, count):
""" General purpose receiver:
Receive exactly #count bytes from #conn """
buf = b''
remaining = count
while remaining > 0:
# Receive part or all of data
tbuf = conn.recv(remaining)
tbuf_len = len(tbuf)
if tbuf_len == 0:
# Really you probably want to return 0 here if buf is empty and
# allow the higher-level routine to determine if the EOF is at
# a proper message boundary in which case, you silently close the
# connection. You would normally only raise an exception if you
# EOF in the *middle* of a message.
raise RuntimeError("end of file")
buf += tbuf
remaining -= tbuf_len
return buf
def receive_proto_block(conn):
""" Receive the next protocol block from #conn. Return a tuple of
request_type (integer) and payload (byte string) """
proto_header = receive_bytes(conn, ProtoHeader.size)
request_type, payload_length = ProtoHeader.unpack(proto_header)
payload = receive_bytes(conn, payload_length)
return request_type, payload
...
request_type, payload = receive_proto_block(conn)
if request_type == LoginRequest:
username = payload.decode()

how do we know the first 4 bytes read on a tcp socket are the length of the message?

The "Sending and receiving logging events across a network" section of the python logging cookbook demonstrates how a clients can send logs via a TCP session.
Log messages are pickled and sent to the server thanks to the socket handler. The server then unpickle the messages and log them.
The code to get a message from the tcp socket is this one:
class LogRecordStreamHandler(SocketServer.StreamRequestHandler):
"""Handler for a streaming logging request.
This basically logs the record using whatever logging policy is
configured locally.
"""
def handle(self):
"""
Handle multiple requests - each expected to be a 4-byte length,
followed by the LogRecord in pickle format. Logs the record
according to whatever policy is configured locally.
"""
while True:
chunk = self.connection.recv(4)
if len(chunk) < 4:
break
slen = struct.unpack('>L', chunk)[0]
chunk = self.connection.recv(slen)
while len(chunk) < slen:
chunk = chunk + self.connection.recv(slen - len(chunk))
obj = self.unPickle(chunk)
record = logging.makeLogRecord(obj)
self.handleLogRecord(record)
# then, methods to handle the record, but that's not the interesting part
class LogRecordSocketReceiver(SocketServer.ThreadingTCPServer):
"""
Simple TCP socket-based logging receiver suitable for testing.
"""
allow_reuse_address = 1
def __init__(self, host='localhost',
port=logging.handlers.DEFAULT_TCP_LOGGING_PORT,
handler=LogRecordStreamHandler):
SocketServer.ThreadingTCPServer.__init__(self, (host, port), handler)
self.abort = 0
self.timeout = 1
self.logname = None
def serve_until_stopped(self):
import select
abort = 0
while not abort:
rd, wr, ex = select.select([self.socket.fileno()], [], [], self.timeout)
if rd:
self.handle_request()
abort = self.abort
What I don't understand in this example is: how do we know that the first 4 bytes we read from the socket constitute the length of the message? I looked at the socket, and logging documentation but could not find mention of it.
Also, the docstring implicitly states that this code is not good enough for production. What is so bad about that code?

As suggested in the comments, the answer is in the sending code (handlers.py in python 2.7.10). I just removed the docstrings/comments that were obvious or irrelevant to this question to make the code more readable.
def makePickle(self, record):
ei = record.exc_info
if ei:
dummy = self.format(record)
record.exc_info = None
d = dict(record.__dict__)
d['msg'] = record.getMessage()
d['args'] = None
s = cPickle.dumps(d, 1)
if ei:
record.exc_info = ei
# slen represents a "long integer", which is usually 32 bits large.
slen = struct.pack(">L", len(s))
# Here is where the 4 byte length is prepended to the message
return slen + s
def emit(self, record):
try:
s = self.makePickle(record)
# s is actually (length of the message) + (message)
self.send(s)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def send(self, s):
"""
Send a pickled string to the socket.
This function allows for partial sends which can happen when the
network is busy.
"""
if self.sock is None:
self.createSocket()
if self.sock:
try:
if hasattr(self.sock, "sendall"):
self.sock.sendall(s)
else:
sentsofar = 0
left = len(s)
while left > 0:
sent = self.sock.send(s[sentsofar:])
sentsofar = sentsofar + sent
left = left - sent
except socket.error:
self.sock.close()
self.sock = None # so we can call createSocket next time

Python Server - Processing WebSocket.close()

I'm working on a small server from the code provided below. The server will interact with a websocket to take a message from a websocket, send a message back to the websocket, and once the websocket calls .close() the server will "shutdown" or close the connection.
However at the moment, when the websocket calls .close() the time it takes to disconnect is long. ( I'm assuming it's the 300 second time out for websockets that occurs when ping/pong messages aren't sent to keep the connection alive). In turn, I think this means that the connection is not closing completely and is just getting lost.
Is there a way to determine and fix this?
I've been reading up on the python socket server documentation and other various sources to figure it out and have come up short. My only solution is the hack comment in the code below.
https://docs.python.org/2/library/socketserver.html
http://pymotw.com/2/SocketServer/
import struct
import SocketServer
from base64 import b64encode
from hashlib import sha1
from mimetools import Message
from StringIO import StringIO
class WebSocketsHandler(SocketServer.StreamRequestHandler):
magic = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'
def setup(self):
SocketServer.StreamRequestHandler.setup(self)
print "connection established", self.client_address
self.handshake_done = False
def handle(self):
while True:
if not self.handshake_done:
self.handshake()
else:
self.read_next_message()
def read_next_message(self):
length = ord(self.rfile.read(2)[1]) & 127
if length == 126:
length = struct.unpack(">H", self.rfile.read(2))[0]
elif length == 127:
length = struct.unpack(">Q", self.rfile.read(8))[0]
masks = [ord(byte) for byte in self.rfile.read(4)]
decoded = ""
for char in self.rfile.read(length):
decoded += chr(ord(char) ^ masks[len(decoded) % 4])
self.on_message(decoded)
def send_message(self, message):
self.request.send(chr(129))
length = len(message)
if length <= 125:
self.request.send(chr(length))
elif length >= 126 and length <= 65535:
self.request.send(chr(126))
self.request.send(struct.pack(">H", length))
else:
self.request.send(chr(127))
self.request.send(struct.pack(">Q", length))
self.request.send(message)
def handshake(self):
data = self.request.recv(1024).strip()
headers = Message(StringIO(data.split('\r\n', 1)[1]))
if headers.get("Upgrade", None) != "websocket":
return
print 'Handshaking...'
key = headers['Sec-WebSocket-Key']
digest = b64encode(sha1(key + self.magic).hexdigest().decode('hex'))
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
self.handshake_done = self.request.send(response)
def on_message(self, message):
print message
self.send_message("How do you do?")
if __name__ == "__main__":
server = SocketServer.TCPServer(
("", 9999), WebSocketsHandler)
server.serve_forever()
[Slightly] modified from: https://gist.github.com/jkp/3136208
NOTE: I am aware of the server.serve_forever() and how that may be an issue. I'm looking for a suggestion or direction along the lines of :
def on_close(): ...

I think you need to implement an onClose() method on your server side. Why you do not use some python framework that has websocket support such as Tornado or Authobahn.
Here is an example for websocket server in python using Tornado
https://developer.mbed.org/cookbook/Websockets-Server
I hope this will help you.