Develop Reference

Python Socket with Multiprocessing and Pickle issue

I am having a Pickle issue with SSL client to server communication using multiprocessing.
I have an SSL client that connects to the server:
SSLClient.py
import socket
import struct
import ssl
import copyreg
from os import path
import socket
import os
from pathlib import Path
from loguru import logger as log
from utils.misc import read_py_config
from datetime import datetime
from cryptography.fernet import Fernet
fernetkey = '1234567'
fernet = Fernet(fernetkey)
class SSLclient:
license = None
licenseencrypted = None
uuid = None
def __init__(self):
try:
path = Path(__file__).parent / "/lcl" #get unique license key
with path.open() as file:
self.licenseencrypted = file.read().rstrip()
self.license = fernet.decrypt(str.encode(self.licenseencrypted)).decode('ascii')
self.host, self.port = "127.0.0.1", 65416
except Exception as e:
log.error("Could not decode license key")
def connect(self):
self.client_crt = os.path.join(os.path.dirname(__file__), 'key/c-.crt')
self.client_key = os.path.join(os.path.dirname(__file__), 'key/ck-.key')
self.server_crt = os.path.join(os.path.dirname(__file__), 'key/s-.crt')
self.sni_hostname = "example.com"
self._context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=self.server_crt)
self._context.load_cert_chain(certfile=self.client_crt, keyfile=self.client_key)
self._sock = None
self._ssock = None
## ---- Client Communication Setup ----
HOST = self.host # The server's hostname or IP address
PORT = self.port # The port used by the server
try:
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._ssock = self._context.wrap_socket(self._sock, server_side=False, server_hostname=self.sni_hostname)
self._ssock.connect((HOST, PORT))
log.info("Socket successfully created")
except socket.error as err:
log.error("socket creation failed with error %s" %(err))
return False
log.info('Waiting for connection')
return True
def closesockconnection(self):
self._ssock.close()
def checkvalidsite(self):
#check if site is active
jsonobj = {
"uuid": self.license,
"ipaddress" : self.external_ip,
"req": "checkvalidsite"
}
send_msg(self._ssock, json.dumps(jsonobj).encode('utf-8'))
active = False
while True:
Response = recv_msg(self._ssock)
if not Response:
return False
if Response is not None:
Response = Response.decode('utf-8')
Response = json.loads(Response)
req = Response['req']
if req == "checkvalidsite":
active = Response['active']
self.info1 = Response['info1']
self.info2 = Response['info2']
return active
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
try:
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
except Exception as e:
log.error("Sending message " + str(e))
def recv_msg(sock: socket): # ---- Use this to receive
try:
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
except Exception as e:
log.error("Receiving message " + str(e))
return False
def recvall(sock: socket, n: int):
try:
# Helper function to receive n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
except Exception as e:
log.error("Receiving all message " + str(e))
raise Exception(e)
I then have a server that is Multithreaded and accepts the connection and communicates with the client.
Server.py
import socket
import os
from socket import AF_INET, SOCK_STREAM, SO_REUSEADDR, SOL_SOCKET, SHUT_RDWR
import ssl
from os import path
from _thread import *
import struct # Here to convert Python data types into byte streams (in string) and back
import traceback
from threading import Thread
import json
import mysql.connector as mysql
import time
from loguru import logger as log
import threading
from cryptography.fernet import Fernet
fernetkey = '12213423423'
fernet = Fernet(fernetkey)
threadLocal = threading.local()
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
try:
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
except Exception as e:
log.error("Error send_msg " + str(e))
def recv_msg(sock: socket): # ---- Use this to receive
try:
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
except Exception as e:
log.error("Receiving message " + str(e))
return False
def recvall(sock: socket, n: int):
try:
# Helper function to receive n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
except Exception as e:
log.error("Receiving all message " + str(e))
raise Exception(e)
# ---- Server Communication Setup
class Newclient:
def __init__(self):
self.addr = None
self.conn = None
self.uuid = None
class Server:
def __init__(self):
self.HOST = '127.0.0.1' # Standard loopback interface address (localhost)
self.PORT = 65416 # Port to listen on (non-privileged ports are > 1023)
self.ThreadCount = 0
self.threads = []
self.sock = None
def checkvalidsite(self, uuid, ipaddress, cursor, db_connection):
sql = "select * from myexample where uuid ='" + uuid + "'"
cursor.execute(sql)
results = cursor.fetchall()
active = False
for row in results:
active = row["active"]
siteid = row["info1"]
clientid = row["info2"]
return active, siteid, clientid
def Serverthreaded_client(self, newclient):
conn = newclient.conn
try:
while True:
# data = conn.recv(2048) # receive message from client
data = recv_msg(conn)
uuid = None
ipaddress = None
req = None
if not data :
return False
if data is not None:
data = json.loads(data.decode('utf-8'))
uuid = data['uuid']
req = data['req']
if uuid is not None and req is not None:
newclient.uuid = uuid
cursor, db_connection = setupDBConnection()
if req == "checkvalidsite":
ipaddress = data['ipaddress']
active, info1, info2 = self.checkvalidsite(uuid, ipaddress, cursor, db_connection)
data = {
"req": "checkvalidsite",
"uuid": uuid,
"active": active,
"info1" : info1,
"info2" : info2
}
if not data:
break
# conn.sendall(str.encode(reply))
send_msg(conn, json.dumps(data).encode('utf-8'))
log.info("Server response sent")
#conn.close()
closeDBConnection(cursor, db_connection)
else:
#send no message
a=1
except Exception as e:
log.warning(str(e))
log.warning(traceback.format_exc())
finally:
log.info("UUID Closing connection")
conn.shutdown(socket.SHUT_RDWR)
conn.close()
#conn.close()
def Serverconnect(self):
try: # create socket
self.server_cert = path.join(path.dirname(__file__), "keys/server.crt")
self.server_key = path.join(path.dirname(__file__), "keys/server.key")
self.client_cert = path.join(path.dirname(__file__), "keys/client.crt")
self._context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
self._context.verify_mode = ssl.CERT_REQUIRED
###self._context.load_cert_chain(self.server_cert, self.server_key)
self._context.load_cert_chain(certfile=self.server_cert, keyfile=self.server_key)
###self._context.load_verify_locations(self.client_cert)
self._context.load_verify_locations(cafile=self.client_cert)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) ###<-- socket.socket() ???
log.info("Socket successfully created")
except socket.error as err:
log.warning("socket creation failed with error %s" %(err))
try: # bind socket to an address
self.sock.bind((self.HOST, self.PORT))
except socket.error as e:
log.warning(str(e))
log.info('Waiting for a Connection..')
self.sock.listen(3)
def Serverwaitforconnection(self):
while True:
Client, addr = self.sock.accept()
conn = self._context.wrap_socket(Client, server_side=True)
log.info('Connected to: ' + addr[0] + ':' + str(addr[1]))
log.info("SSL established. Peer: {}".format(conn.getpeercert()))
newclient = Newclient()
newclient.addr = addr
newclient.conn = conn
thread = Thread(target=self.Serverthreaded_client, args =(newclient, ))
thread.start()
self.threads.append(newclient)
self.ThreadCount += 1
log.info('Thread Number: ' + str(self.ThreadCount))
def startserver():
server = Server()
server.Serverconnect()
server.Serverwaitforconnection()
serverthread = Thread(target=startserver)
serverthread.daemon = False
serverthread.start()
The server accepts the connection with SSL then waits for a message. It investigates the message command, executes the respective function and returns the data from the database as a response (checkvalidsite in this example).
All good so far (as far as I can tell).
I also have the main program that calls the SSLClient and connects.
Main program
remoteclient = SSLclient()
successfulconnection = remoteclient.connect()
siteactive = remoteclient.checkvalidsite()
So far all is well. However I also have the main program reading in frames from multiple cameras. Can be 20 cameras for example. In order to do this I created multiprocessing to deal with the camera load. Each camera or two cameras per, are assigned to a processor (depending on the number of cores in the machine).
(code below has been stripped out to simplify reading)
x = range(3, 6)
for n in x:
processes = multiprocessing.Process(target=activateMainProgram, args=(queue1, queue2, queue3, queue4, remoteclient, ))
processes.start()
When I try pass the remoteclient (SSLClient) as an argument I get the error:
cannot pickle 'SSLContext' object
I then (after reading online) added the code to the SSLClient:
def save_sslcontext(obj):
return obj.__class__, (obj.protocol,)
copyreg.pickle(ssl.SSLContext, save_sslcontext)
but then I get the error:
cannot pickle 'SSLContext' object
There are 2 options I experimented with:
Trying to get the pickle working (which would be ideal) as the processes themselves each need to communicate with the server. So the processes need to call functions from the SSLClient file. But I cannot get over the pickle issue and can't find a solution online
I then placed the remoteclient = SSLClient code outside the main function. Hoping it would run first and then be accessible to the processes. This worked, however what I learnt was that when a process is called (as it does not share memory) it reprocesses the entire file. Meaning if I have 10 processes each with 2 cameras then I would have 10 connections to the server (1 per process). This means on the server side I would also have 10 threads running each connection. Though it works, it seems significantly inefficient.
Being a noob and self taught in Python I am not sure how to resolve the issue and after 3 days, I figured I would reach out for assistance. If I could get assistance with the pickle issue of the SSLClient then I will have one connection that is shared with all processes and 1 thread in the server to deal with them.
P.s. I have cobbled all of the code together myself and being new to Python if you see that I am totally going down the wrong, incorrect, non-professional track, feel free to yell.
Much appreciated.
Update:
If I change the SSLClient code to:
def save_sslcontext(obj):
return obj.__class__, (obj.protocol,)
copyreg.pickle(ssl.SSLContext, save_sslcontext)
Then I get the error:
[WinError 10038] An operation was attempted on something that is not a socket
Not sure what is better..

How do I run python-osc inside OBS as script?

I want/need to make this code run inside OBS as a script. I took this OSC Sender for OBS that it does what the name says: it sends OSC messages. But, inside the code, the server part is commented, because it doesn't work... right out of the box. It is needed to run OBS with LD_PRELOAD=/path/libpython3.7.so.
I modified the code but (uncommented lines) but, when server is opened, OBS stays unusable, blocked. So, I tried the "Async Server" - "Concurrent Mode" of python-osc. I take the example from its wiki (that it works in a console) and I mixed with the other script. I made a nice code... but it makes something weird. This is the code:
"""
"""
# osc data
import argparse
import random
import time
import math
import asyncio
from pythonosc import osc_message_builder
from pythonosc import udp_client
from pythonosc.dispatcher import Dispatcher
from pythonosc.osc_server import AsyncIOOSCUDPServer
client = None
server = None
#obs !
import obspython as obs
pleaseLog = False
host = None
serverPort = None
# transport = None
async def loop():
"""Example main loop that only runs for 10 iterations before finishing"""
for i in range(10):
print(f"Loop {i}")
await asyncio.sleep(.5)
async def init_main():
# global transport
server = AsyncIOOSCUDPServer((host, serverPort), dispatcher, asyncio.get_event_loop())
transport, protocol = await server.create_serve_endpoint() # Create datagram endpoint and start serving
await loop() # Enter main loop of program
transport.close() # Clean up serve endpoint
def handleOSC(*args):
for i in args:
print(i)
# defines script description
def script_description():
return '''Send OSC data when source is activated if source name begins with /'''
# defines user properties
def script_properties():
global props
props = obs.obs_properties_create()
obs.obs_properties_add_text(props, "host", "Host IP", obs.OBS_TEXT_DEFAULT)
obs.obs_properties_add_int(props, "clientPort", "Host port", 1, 400000, 1)
obs.obs_properties_add_bool(props, "logOscOutput", "Log OSC output")
obs.obs_properties_add_int(props, "serverPort", "Listen port", 1, 400000, 1)
return props
def script_defaults(settings):
obs.obs_data_set_default_string(settings, "host", "127.0.0.1")
obs.obs_data_set_default_int(settings, "clientPort", 10000)
obs.obs_data_set_default_int(settings, "serverPort", 10001)
def source_activated(cd):
global pleaseLog
source = obs.calldata_source(cd, "source")
if source is not None:
name = obs.obs_source_get_name(source)
if name[0] == "/":
client.send_message(name, 1)
if (pleaseLog):
print("send " + name)
def script_load(settings):
global dispatcher
global host
sh = obs.obs_get_signal_handler()
obs.signal_handler_connect(sh, "source_activate", source_activated)
dispatcher = Dispatcher()
dispatcher.map("/*", handleOSC)
# def script_unload():
# global transport
# print(f'script_unload(settings)')
# transport.close() # Clean up serve endpoint
def script_update(settings):
global host
global client
global server
global clientPort
global serverPort
global pleaseLog
pleaseLog = obs.obs_data_get_bool(settings, "logOscOutput")
host = obs.obs_data_get_string(settings, "host")
clientPort = obs.obs_data_get_int(settings, "clientPort")
serverPort = obs.obs_data_get_int(settings, "serverPort")
# Client
client = udp_client.SimpleUDPClient(host, clientPort)
print("target set to "+host+":"+str(clientPort)+"")
# Server
print("serving in "+host+":"+str(serverPort)+"")
asyncio.run(init_main())
I'm not a programmer, so asyncio is a mess for me. I don't understand it. But I know what it does and what I'd like it to do.
When the script is loaded (in script_update(settings)), it opens the server and run loop() function. This function is a 5 seconds loop that prints some text. If I connect to the correct port from PureData and send some OSC messages, these messages arrives to OBS... but all of them drops together when the loop finishes. Meanwhile, there's nothing, OBS is blocked and nothing is in Script Log.
If I run the python-osc asyncio example code in a console, while loop is being executed, every Loop 0, Loop 1, etc, and every message arrives and all of them are printed at the right time.
How should I get this code to work? I need to open that server and run a much more code at the same time. I'm using OBS like a game engine.

Finally, I didn't use Asyncio. Instead of that, I use the Blocking Server method but inside a threading.Thread() function. For the moment, it is working right in this way.
It's important to close port when unload or reload script.
Oro, Treblig_Punisher (users from OBS Discor) helped me to achieve this.
"""
"""
# osc data
import argparse, random, time, math, threading
from pythonosc import osc_message_builder
from pythonosc import udp_client
from pythonosc import dispatcher
from pythonosc import osc_server
targetIp = "127.0.0.1"
targetPort = 10000
serverPort = 10008
client = None
server = None
#obs !
import obspython as obs
pleaseLog = False
def handleOSC(address, args, data):
print (address)
print (args)
print (data)
# defines script description
def script_description():
return '''Send and receive OSC messages'''
# defines user properties
def script_properties():
#global props
props = obs.obs_properties_create()
obs.obs_properties_add_text(props, "host", "Host IP", obs.OBS_TEXT_DEFAULT)
obs.obs_properties_add_int(props, "port", "Host port", 1, 400000, 1)
obs.obs_properties_add_bool(props, "logOscOutput", "Log OSC output")
obs.obs_properties_add_int(props, "serverPort", "Listen port", 1, 400000, 1)
return props
def script_defaults(settings):
obs.obs_data_set_default_string(settings, "host", targetIp)
obs.obs_data_set_default_int(settings, "port", targetPort)
obs.obs_data_set_default_int(settings, "serverPort", serverPort)
# Cuando se activa un source
def source_activated(cd):
global pleaseLog
source = obs.calldata_source(cd, "source")
if source is not None:
name = obs.obs_source_get_name(source)
current_scene = obs.obs_scene_from_source(obs.obs_frontend_get_current_scene())
scene_item = obs.obs_scene_find_source(current_scene, name)
boolean = obs.obs_sceneitem_visible(scene_item)
print(boolean)
if name[0] == "/":
client.send_message(name, boolean)
if (pleaseLog):
print("send " + name + boolean)
def script_load(settings):
global despachante
sh = obs.obs_get_signal_handler()
obs.signal_handler_connect(sh, "source_activate", source_activated)
despachante = dispatcher.Dispatcher()
despachante.map("/*", handleOSC)
def script_unload():
global server
server.server_close()
def script_update(settings):
global host
global port
global client
global server
global pleaseLog
pleaseLog = obs.obs_data_get_bool(settings, "logOscOutput")
host = obs.obs_data_get_string(settings, "host")
port = obs.obs_data_get_int(settings, "port")
# Client
client = udp_client.SimpleUDPClient(host, port)
print("target set to "+host+":"+str(port)+"")
# Server
serverPort = obs.obs_data_get_int(settings, "serverPort")
try:
server.server_close()
except:
print('*Server aun no creado')
# raise
server = osc_server.BlockingOSCUDPServer(("127.0.0.1", serverPort), despachante)
t = threading.Thread(target=server_th, args=([settings]))
t.daemon = True
t.start()
# Loop every 1000ms
# obs.timer_remove(funcion_loop)
# if and source_name != "":
# obs.timer_add(funcion_loop, 1000)
# Threading function
def server_th(settings):
print(f'Serving on {server.server_address}')
server.serve_forever() # Blocks forever

Imported module logging calls not showing up

I have been reading up on proper logging and so far I am liking how it is going. All was fine until I tried to do logging in a main file and a module I wrote. The main file is able to write to a file and the console but the imported module displays nothing in either. If i had to take a guess, I am assuming I would have to configure the modules output separately as I am using in code configs. Problem is I am not sure how or if that is even the reason. I have tried my best to google this instead of asking but here I am now. Here is the link to the source code. If you try to run it you may have to change the import as pycharm does not like it when I import a file directly. So from "from tests import speedtest" to "import speedtest" The files are main.py and speedtest.py
Main
import logging
from tests import speedtest
import time
# Logging configuration
logFormatter = logging.Formatter("%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s")
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
# creates a handler to deal with writing to the file
file_handler = logging.FileHandler("log.txt", mode="w")
file_handler.setFormatter(logFormatter)
# handler for writing to the console
console_handler = logging.StreamHandler()
console_handler.setFormatter(logFormatter)
# adds the handlers to the root logger
logger.addHandler(file_handler)
logger.addHandler(console_handler)
# max speed provided
NOMINAL_SPEED = 50
# threshold in percentage 60% seems to be a decent amount to expect.
THRESHOLD = 60
# padding in percentage for severe warning
PAD = 10
# amount of time in between runs
INTERVAL = 300
class Main:
"""
Main running class
"""
def __init__(self):
self.speedtest = speedtest.SpeedTest(share=True)
self.threshold = THRESHOLD
self.pad = PAD
self.nominal = NOMINAL_SPEED
self.done = False
logger.debug("Starting main loop.")
while not self.done:
self.loop()
time.sleep(INTERVAL)
def loop(self):
try:
results = self.speedtest.run()
except Exception as e:
logger.error("Skipped running speed test this run. Will try again next time")
return
download = float(results["download"][:-7])
upload = float(results["upload"][:-7])
url = results["url"]
host = results["host"]
diff_download = (download / self.nominal) * 100
logger.debug("Current download is {} Mbps upload is {} Mbps. Share url: {} host: {}".format(download, upload, url, host))
if (((self.threshold - self.pad)/100) * self.nominal) <= diff_download <= ((self.threshold/100) * self.nominal):
logger.info("Speed is currently at {}% nominal.".format(diff_download))
self.warning()
elif diff_download <= ((self.threshold - self.pad)/100) * self.nominal:
logger.info("Speed is currently at {}% nominal. This is a problem.".format(diff_download))
self.critical()
def warning(self):
pass
def critical(self):
pass
if __name__ == "__main__":
Main()
speedtest
import subprocess
import logging
import os
class SpeedTest:
"""
Class to run speed test and return the results in an easy to use manner
"""
def __init__(self, share=False):
"""
Init method
:param share: When set to true it will also return a url to the speed test image
:return:
"""
self.logger = logging.getLogger(__name__)
self.logger.addHandler(logging.NullHandler())
self._share = share
if share is True:
self.logger.debug("Share flag set to True")
self.cmd = ["speedtest-cli", "--share"]
else:
self.logger.debug("Share not set to true. Ignoring share url")
self.cmd = ["speedtest-cli"]
def run(self):
"""
Runs the speed test returning a dict containing upload, download, ping, and share url if wanted.
:return:
"""
self.logger.debug("Starting speedtest!")
# check_output returns the output in bytes so we use decode() to turn it into a simple string. Then we split
# the lines giving us a list.
try:
stdout = subprocess.check_output(self.cmd).decode().splitlines()
except subprocess.CalledProcessError as e:
self.logger.error(e)
raise e
res = {}
for i in stdout:
if "Download:" in i:
res["download"] = i[10:]
if "Upload:" in i:
res["upload"] = i[8:]
if "Hosted" in i:
res["host"] = i[2:]
if self._share is True and "Share results:" in i:
res["url"] = i[15:]
else:
res["url"] = None
return res
def ping(self, addr):
"""
Pings an address and returns a 1 if the connection can not be made or a 0 if it succeeds
:param addr: IPv4 address
:return:
"""
try:
if os.name is "nt":
self.logger.debug("Windows OS detected")
self.logger.info("Pinging {}".format(addr))
subprocess.check_output(["ping", "-n", "1", addr])
elif os.name is "posix":
self.logger.debug("Nix OS detected")
subprocess.check_output(["ping", "-c", "1", addr])
except subprocess.CalledProcessError:
self.logger.warning("Returned non zero value. Is the internet working?")
return 1
return 0
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
for i in SpeedTest(share=True).run().items():
print(i)
print(SpeedTest().ping("8.8.8.0"))

In speedtest.py when you call:
logging.getLogger(__name__)
it will create a logger object for speedtest.py, so you will have to configure it separately. If you want it to be the same logger as in the main just add:
self.speedtest.logger = logger
after you create the SpeedTest object in Main's constructor
Another option for you is to pass __name__ as an argument to SpeedTest() and create the logger with that argument (I think this is a better option for you since you write to the logger in the constructor).

how do we know the first 4 bytes read on a tcp socket are the length of the message?

The "Sending and receiving logging events across a network" section of the python logging cookbook demonstrates how a clients can send logs via a TCP session.
Log messages are pickled and sent to the server thanks to the socket handler. The server then unpickle the messages and log them.
The code to get a message from the tcp socket is this one:
class LogRecordStreamHandler(SocketServer.StreamRequestHandler):
"""Handler for a streaming logging request.
This basically logs the record using whatever logging policy is
configured locally.
"""
def handle(self):
"""
Handle multiple requests - each expected to be a 4-byte length,
followed by the LogRecord in pickle format. Logs the record
according to whatever policy is configured locally.
"""
while True:
chunk = self.connection.recv(4)
if len(chunk) < 4:
break
slen = struct.unpack('>L', chunk)[0]
chunk = self.connection.recv(slen)
while len(chunk) < slen:
chunk = chunk + self.connection.recv(slen - len(chunk))
obj = self.unPickle(chunk)
record = logging.makeLogRecord(obj)
self.handleLogRecord(record)
# then, methods to handle the record, but that's not the interesting part
class LogRecordSocketReceiver(SocketServer.ThreadingTCPServer):
"""
Simple TCP socket-based logging receiver suitable for testing.
"""
allow_reuse_address = 1
def __init__(self, host='localhost',
port=logging.handlers.DEFAULT_TCP_LOGGING_PORT,
handler=LogRecordStreamHandler):
SocketServer.ThreadingTCPServer.__init__(self, (host, port), handler)
self.abort = 0
self.timeout = 1
self.logname = None
def serve_until_stopped(self):
import select
abort = 0
while not abort:
rd, wr, ex = select.select([self.socket.fileno()], [], [], self.timeout)
if rd:
self.handle_request()
abort = self.abort
What I don't understand in this example is: how do we know that the first 4 bytes we read from the socket constitute the length of the message? I looked at the socket, and logging documentation but could not find mention of it.
Also, the docstring implicitly states that this code is not good enough for production. What is so bad about that code?

As suggested in the comments, the answer is in the sending code (handlers.py in python 2.7.10). I just removed the docstrings/comments that were obvious or irrelevant to this question to make the code more readable.
def makePickle(self, record):
ei = record.exc_info
if ei:
dummy = self.format(record)
record.exc_info = None
d = dict(record.__dict__)
d['msg'] = record.getMessage()
d['args'] = None
s = cPickle.dumps(d, 1)
if ei:
record.exc_info = ei
# slen represents a "long integer", which is usually 32 bits large.
slen = struct.pack(">L", len(s))
# Here is where the 4 byte length is prepended to the message
return slen + s
def emit(self, record):
try:
s = self.makePickle(record)
# s is actually (length of the message) + (message)
self.send(s)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def send(self, s):
"""
Send a pickled string to the socket.
This function allows for partial sends which can happen when the
network is busy.
"""
if self.sock is None:
self.createSocket()
if self.sock:
try:
if hasattr(self.sock, "sendall"):
self.sock.sendall(s)
else:
sentsofar = 0
left = len(s)
while left > 0:
sent = self.sock.send(s[sentsofar:])
sentsofar = sentsofar + sent
left = left - sent
except socket.error:
self.sock.close()
self.sock = None # so we can call createSocket next time

I/O intensive serial port application: Porting from Threading, Queue based design to Asynchronous (ala Twisted)

So, I've been working on an application for a client that communicates with wireless devices via a Serial (RS-232) "Master". I've currently written the core of the app using threading (below). I've been noticing on #python that the consensus seems to be to NOT use threads and to use Twisted's asynchronous communication abilities.
I haven't been able to find any good examples of using twisted for serial port async I/O communication. However, I have found Dave Peticolas' 'Twisted Introduction' (thanks nosklo) that I'm currently working through, but, it uses sockets instead of serial communication (but the async concept is definitely very well explained).
How would I go about porting this app over to Twisted from using Threading, Queues? Are there any advantages/disadvantages (I have noticed that, on occasion, if a thread hangs it will BSOD the system)?
The Code (msg_poller.py)
from livedatafeed import LiveDataFeed
from msg_build import build_message_to_send
from utils import get_item_from_queue
from protocol_wrapper import ProtocolWrapper, ProtocolStatus
from crc16 import *
import time
import Queue
import threading
import serial
import gc
gc.enable()
PROTOCOL_HEADER = '\x01'
PROTOCOL_FOOTER = '\x0D\x0A'
PROTOCOL_DLE = '\x90'
INITIAL_MODBUS = 0xFFFF
class Poller:
"""
Connects to the serial port and polls nodes for data.
Reads response from node(s) and loads that data into queue.
Parses qdata and writes that data to database.
"""
def __init__(self,
port,
baudrate,
parity,
rtscts,
xonxoff,
echo=False):
try:
self.serial = serial.serial_for_url(port,
baudrate,
parity=parity,
rtscts=rtscts,
xonxoff=xonxoff,
timeout=.01)
except AttributeError:
self.serial = serial.Serial(port,
baudrate,
parity=parity,
rtscts=rtscts,
xonxoff=xonxoff,
timeout=.01)
self.com_data_q = None
self.com_error_q = None
self.livefeed = LiveDataFeed()
self.timer = time.time()
self.dtr_state = True
self.rts_state = True
self.break_state = False
def start(self):
self.data_q = Queue.Queue()
self.error_q = Queue.Queue()
com_error = get_item_from_queue(self.error_q)
if com_error is not None:
print 'Error %s' % (com_error)
self.timer = time.time()
self.alive = True
# start monitor thread
#
self.mon_thread = threading.Thread(target=self.reader)
self.mon_thread.setDaemon(1)
self.mon_thread.start()
# start sending thread
#
self.trans_thread = threading.Thread(target=self.writer)
self.trans_thread.setDaemon(1)
self.trans_thread.start()
def stop(self):
try:
self.alive = False
self.serial.close()
except (KeyboardInterrupt, SystemExit):
self.alive = False
def reader(self):
"""
Reads data from the serial port using self.mon_thread.
Displays that data on the screen.
"""
from rmsg_format import message_crc, message_format
while self.alive:
try:
while self.serial.inWaiting() != 0:
# Read node data from the serial port. Data should be 96B.
data = self.serial.read(96)
data += self.serial.read(self.serial.inWaiting())
if len(data) > 0:
# Put data in to the data_q object
self.data_q.put(data)
if len(data) == 96:
msg = self.data_q.get()
pw = ProtocolWrapper(
header=PROTOCOL_HEADER,
footer=PROTOCOL_FOOTER,
dle=PROTOCOL_DLE)
status = map(pw.input, msg)
if status[-1] == ProtocolStatus.IN_MSG:
# Feed all the bytes of 'msg' sequentially into pw.input
# Parse the received CRC into a 16-bit integer
rec_crc = message_crc.parse(msg[-4:]).crc
# Compute the CRC on the message
calc_crc = calcString(msg[:-4], INITIAL_MODBUS)
from datetime import datetime
ts = datetime.now().strftime('%Y/%m/%d %H:%M:%S')
if rec_crc != calc_crc:
print ts
print 'ERROR: CRC Mismatch'
print msg.encode('hex')
else:
#msg = message_format.parse(msg[1:])
#print msg.encode('hex') + "\r\n"
msg = message_format.parse(msg[1:])
print msg
#return msg
gc.collect()
time.sleep(.2)
except (KeyboardInterrupt, SystemExit, Exception, TypeError):
self.alive = False
self.serial.close()
raise
def writer(self):
"""
Builds the packet to poll each node for data.
Writes that data to the serial port using self.trans_thread
"""
import time
try:
while self.alive:
try:
dest_module_code = ['DRILLRIG',
'POWERPLANT',
'GENSET',
'MUDPUMP']
dest_ser_no = lambda x: x + 1
for code in dest_module_code:
if code != 'POWERPLANT':
msg = build_message_to_send(
data_len=0x10,
dest_module_code='%s' % (code),
dest_ser_no=dest_ser_no(0),
dest_customer_code='*****',
ret_ser_no=0x01,
ret_module_code='DOGHOUSE',
ret_customer_code='*****',
command='POLL_NODE',
data=[])
self.serial.write(msg)
time.sleep(.2)
gc.collect()
elif code == 'POWERPLANT':
msg = build_message_to_send(
data_len=0x10,
dest_module_code='POWERPLANT',
dest_ser_no=dest_ser_no(0),
dest_customer_code='*****',
ret_ser_no=0x01,
ret_module_code='DOGHOUSE',
ret_customer_code='*****',
command='POLL_NODE',
data=[])
self.serial.write(msg)
time.sleep(.2)
gc.collect()
msg = build_message_to_send(
data_len=0x10,
dest_module_code='POWERPLANT',
dest_ser_no=dest_ser_no(1),
dest_customer_code='*****',
ret_ser_no=0x01,
ret_module_code='DOGHOUSE',
ret_customer_code='*****',
command='POLL_NODE',
data=[])
self.serial.write(msg)
time.sleep(.2)
gc.collect()
except (KeyboardInterrupt, SystemExit):
self.alive = False
self.serial.close()
raise
except (KeyboardInterrupt, SystemExit):
self.alive = False
self.serial.close()
raise
def main():
poller = Poller(
port='COM4',
baudrate=115200,
parity=serial.PARITY_NONE,
rtscts=0,
xonxoff=0,
)
poller.start()
poller.reader()
poller.writer()
poller.stop()
if __name__ == '__main__':
main()

It is very difficult (if not impossible) to write a direct one-to-one mapping program between threading/queue approach and one that uses twisted.
I would suggest that, get a hang of twisted and its reactor way it's use of Protocol and the protocol specific methods. Think about it as as all the asynchronous things that you had been explicitly coding using threads and queues are given to you for free when you are using deferred using twisted.
twisted does seem to support SerialPort over it's reactor using SerialPort transport class and the basic structure seems to be somewhat like this.
from twisted.internet import reactor
from twisted.internet.serialport import SerialPort
SerialPort(YourProtocolClass(), Port, reactor, baudrate=baudrate))
reactor.run()
In YourProtocolClass() would you handle the various events that are specific to your Serial Port Communication requirements. The doc/core/examples directory contains examples such as gpsfix.py and mouse.py.