I'm trying to warp Pyro's name server into a more convenient object that would allow me to start and stop it as I wish. For example, I would like to be able to do something like
nameServer = NameServer("localhost")
nameServer.startNS()
[... make some other operations...]
nameServer.stopNS()
nameServer = None
[... make some other operations...]
nameServer = NameServer("localhost")
nameServer.startNS()
using the following definition for the NameServer class:
class NameServer(threadutil.Thread):
def __init__(self, host, isDeamon=True, port=None, enableBroadcast=True,
bchost=None, bcport=None, unixsocket=None, nathost=None, natport=None):
super(NameServer,self).__init__()
self.setDaemon(isDeamon)
self.host=host
self.started=threadutil.Event()
self.unixsocket = unixsocket
self.port = port
self.enableBroadcast = enableBroadcast
self.bchost = bchost
self.bcport = bcport
self.nathost = nathost
self.natport = natport
"""
This code is taken from Pyro4.naming.startNSloop
"""
self.ns_daemon = naming.NameServerDaemon(self.host, self.port, self.unixsocket,
nathost=self.nathost, natport=self.natport)
self.uri = self.ns_daemon.uriFor(self.ns_daemon.nameserver)
internalUri = self.ns_daemon.uriFor(self.ns_daemon.nameserver, nat=False)
self.bcserver=None
if self.unixsocket:
hostip = "Unix domain socket"
else:
hostip = self.ns_daemon.sock.getsockname()[0]
if hostip.startswith("127."):
enableBroadcast=False
if enableBroadcast:
# Make sure to pass the internal uri to the broadcast responder.
# It is almost always useless to let it return the external uri,
# because external systems won't be able to talk to this thing anyway.
bcserver=naming.BroadcastServer(internalUri, self.bchost, self.bcport)
bcserver.runInThread()
def run(self):
try:
self.ns_daemon.requestLoop()
finally:
self.ns_daemon.close()
if self.bcserver is not None:
self.bcserver.close()
def startNS(self):
self.start()
def stopNS(self):
self.ns_daemon.shutdown()
if self.bcserver is not None:
self.bcserver.shutdown()
So far, so good. It works as expected. However, if I run a command Pyro4.naming.locateNS() from another thread when the name server is running, then the next time I call nameServer.stopNS(), the program freezes. Anyone has an idea why? And what would be the best (at least a better) way to write such a NameServer wrapper.
There is an example in the Pyro4 repository that you could adapt.
https://github.com/delmic/Pyro4/blob/master/examples/eventloop/server.py
from __future__ import print_function
import socket
import select
import sys
import Pyro4.core
import Pyro4.naming
if sys.version_info<(3,0):
input=raw_input
print("Make sure that you don't have a name server running already.")
servertype=input("Servertype thread/multiplex (t/m)?")
if servertype=='t':
Pyro4.config.SERVERTYPE="thread"
else:
Pyro4.config.SERVERTYPE="multiplex"
hostname=socket.gethostname()
class EmbeddedServer(object):
def multiply(self, x, y):
return x*y
print("initializing services... servertype=%s" % Pyro4.config.SERVERTYPE)
# start a name server with broadcast server as well
nameserverUri, nameserverDaemon, broadcastServer = Pyro4.naming.startNS(host=hostname)
assert broadcastServer is not None, "expect a broadcast server to be created"
print("got a Nameserver, uri=%s" % nameserverUri)
print("ns daemon location string=%s" % nameserverDaemon.locationStr)
print("ns daemon sockets=%s" % nameserverDaemon.sockets)
print("bc server socket=%s (fileno %d)" % (broadcastServer.sock, broadcastServer.fileno()))
# create a Pyro daemon
pyrodaemon=Pyro4.core.Daemon(host=hostname)
print("daemon location string=%s" % pyrodaemon.locationStr)
print("daemon sockets=%s" % pyrodaemon.sockets)
# register a server object with the daemon
serveruri=pyrodaemon.register(EmbeddedServer())
print("server uri=%s" % serveruri)
# register it with the embedded nameserver directly
nameserverDaemon.nameserver.register("example.embedded.server",serveruri)
print("")
# below is our custom event loop.
while True:
print("Waiting for events...")
# create sets of the socket objects we will be waiting on
# (a set provides fast lookup compared to a list)
nameserverSockets = set(nameserverDaemon.sockets)
pyroSockets = set(pyrodaemon.sockets)
rs=[broadcastServer] # only the broadcast server is directly usable as a select() object
rs.extend(nameserverSockets)
rs.extend(pyroSockets)
rs,_,_ = select.select(rs,[],[],3)
eventsForNameserver=[]
eventsForDaemon=[]
for s in rs:
if s is broadcastServer:
print("Broadcast server received a request")
broadcastServer.processRequest()
elif s in nameserverSockets:
eventsForNameserver.append(s)
elif s in pyroSockets:
eventsForDaemon.append(s)
if eventsForNameserver:
print("Nameserver received a request")
nameserverDaemon.events(eventsForNameserver)
if eventsForDaemon:
print("Daemon received a request")
pyrodaemon.events(eventsForDaemon)
nameserverDaemon.close()
broadcastServer.close()
pyrodaemon.close()
print("done")
Related
The bounty expires in 5 days. Answers to this question are eligible for a +50 reputation bounty.
Haley Mueller wants to draw more attention to this question.
I'm new to Python so this could be a simple fix.
I am using Flask and sockets for this Python project. I am starting the socket on another thread so I can actively listen for new messages. I have an array variable called 'SocketConnections' that is within my UdpComms class. The variable gets a new 'Connection' appended to it when a new socket connection is made. That works correctly. My issue is that when I try to read 'SocketConnections' from outside of the thread looking in, it is an empty array.
server.py
from flask import Flask, jsonify
import UdpComms as U
app = Flask(__name__)
#app.route('/api/talk', methods=['POST'])
def talk():
global global_server_socket
apples = global_server_socket.SocketConnections
return jsonify(message=apples)
global_server_socket = None
def start_server():
global global_server_socket
sock = U.UdpComms(udpIP="127.0.0.1", portTX=8000, portRX=8001, enableRX=True, suppressWarnings=True)
i = 0
global_server_socket = sock
while True:
i += 1
data = sock.ReadReceivedData() # read data
if data != None: # if NEW data has been received since last ReadReceivedData function call
print(data) # print new received data
time.sleep(1)
if __name__ == '__main__':
server_thread = threading.Thread(target=start_server)
server_thread.start()
app.run(debug=True,host='192.168.0.25')
UdpComms.py
import json
import uuid
class UdpComms():
def __init__(self,udpIP,portTX,portRX,enableRX=False,suppressWarnings=True):
self.SocketConnections = []
import socket
self.udpIP = udpIP
self.udpSendPort = portTX
self.udpRcvPort = portRX
self.enableRX = enableRX
self.suppressWarnings = suppressWarnings # when true warnings are suppressed
self.isDataReceived = False
self.dataRX = None
# Connect via UDP
self.udpSock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # internet protocol, udp (DGRAM) socket
self.udpSock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # allows the address/port to be reused immediately instead of it being stuck in the TIME_WAIT state waiting for late packets to arrive.
self.udpSock.bind((udpIP, portRX))
# Create Receiving thread if required
if enableRX:
import threading
self.rxThread = threading.Thread(target=self.ReadUdpThreadFunc, daemon=True)
self.rxThread.start()
def __del__(self):
self.CloseSocket()
def CloseSocket(self):
# Function to close socket
self.udpSock.close()
def SendData(self, strToSend):
# Use this function to send string to C#
self.udpSock.sendto(bytes(strToSend,'utf-8'), (self.udpIP, self.udpSendPort))
def SendDataAddress(self, strToSend, guid):
# Use this function to send string to C#
print('finding connection: ' + guid)
if self.SocketConnections:
connection = self.GetConnectionByGUID(guid)
print('found connection: ' + guid)
if connection is not None:
self.udpSock.sendto(bytes(strToSend,'utf-8'), connection.Address)
def ReceiveData(self):
if not self.enableRX: # if RX is not enabled, raise error
raise ValueError("Attempting to receive data without enabling this setting. Ensure this is enabled from the constructor")
data = None
try:
data, _ = self.udpSock.recvfrom(1024)
print('Socket data recieved from: ', _)
if self.IsNewConnection(_) == True:
print('New socket')
self.SendDataAddress("INIT:" + self.SocketConnections[-1].GUID, self.SocketConnections[-1].GUID)
data = data.decode('utf-8')
except WindowsError as e:
if e.winerror == 10054: # An error occurs if you try to receive before connecting to other application
if not self.suppressWarnings:
print("Are You connected to the other application? Connect to it!")
else:
pass
else:
raise ValueError("Unexpected Error. Are you sure that the received data can be converted to a string")
return data
def ReadUdpThreadFunc(self): # Should be called from thread
self.isDataReceived = False # Initially nothing received
while True:
data = self.ReceiveData() # Blocks (in thread) until data is returned (OR MAYBE UNTIL SOME TIMEOUT AS WELL)
self.dataRX = data # Populate AFTER new data is received
self.isDataReceived = True
# When it reaches here, data received is available
def ReadReceivedData(self):
data = None
if self.isDataReceived: # if data has been received
self.isDataReceived = False
data = self.dataRX
self.dataRX = None # Empty receive buffer
if data != None and data.startswith('DIALOG:'): #send it info
split = data.split(':')[1]
return data
class Connection:
def __init__(self, gUID, address) -> None:
self.GUID = gUID
self.Address = address
def IsNewConnection(self, address):
for connection in self.SocketConnections:
if connection.Address == address:
return False
print('Appending new connection...')
connection = self.Connection(str(uuid.uuid4()),address)
self.SocketConnections.append(connection)
return True
def GetConnectionByGUID(self, guid):
for connection in self.SocketConnections:
if connection.GUID == guid:
return connection
return None
As mentioned above. When IsNewConnection() is called in UdpComms it does append a new object to SocketConnections. It's just trying to view the SocketConnections in the app.route that is empty. My plans are to be able to send socket messages from the app.routes
For interprocess communication you may try to use something like shared memory documented here
Instead of declaring your self.SocketConnections as a list = []
you'd use self.SocketConnections = Array('i', range(10)) (you are then limited to remembering only 10 connections though).
I want/need to make this code run inside OBS as a script. I took this OSC Sender for OBS that it does what the name says: it sends OSC messages. But, inside the code, the server part is commented, because it doesn't work... right out of the box. It is needed to run OBS with LD_PRELOAD=/path/libpython3.7.so.
I modified the code but (uncommented lines) but, when server is opened, OBS stays unusable, blocked. So, I tried the "Async Server" - "Concurrent Mode" of python-osc. I take the example from its wiki (that it works in a console) and I mixed with the other script. I made a nice code... but it makes something weird. This is the code:
"""
"""
# osc data
import argparse
import random
import time
import math
import asyncio
from pythonosc import osc_message_builder
from pythonosc import udp_client
from pythonosc.dispatcher import Dispatcher
from pythonosc.osc_server import AsyncIOOSCUDPServer
client = None
server = None
#obs !
import obspython as obs
pleaseLog = False
host = None
serverPort = None
# transport = None
async def loop():
"""Example main loop that only runs for 10 iterations before finishing"""
for i in range(10):
print(f"Loop {i}")
await asyncio.sleep(.5)
async def init_main():
# global transport
server = AsyncIOOSCUDPServer((host, serverPort), dispatcher, asyncio.get_event_loop())
transport, protocol = await server.create_serve_endpoint() # Create datagram endpoint and start serving
await loop() # Enter main loop of program
transport.close() # Clean up serve endpoint
def handleOSC(*args):
for i in args:
print(i)
# defines script description
def script_description():
return '''Send OSC data when source is activated if source name begins with /'''
# defines user properties
def script_properties():
global props
props = obs.obs_properties_create()
obs.obs_properties_add_text(props, "host", "Host IP", obs.OBS_TEXT_DEFAULT)
obs.obs_properties_add_int(props, "clientPort", "Host port", 1, 400000, 1)
obs.obs_properties_add_bool(props, "logOscOutput", "Log OSC output")
obs.obs_properties_add_int(props, "serverPort", "Listen port", 1, 400000, 1)
return props
def script_defaults(settings):
obs.obs_data_set_default_string(settings, "host", "127.0.0.1")
obs.obs_data_set_default_int(settings, "clientPort", 10000)
obs.obs_data_set_default_int(settings, "serverPort", 10001)
def source_activated(cd):
global pleaseLog
source = obs.calldata_source(cd, "source")
if source is not None:
name = obs.obs_source_get_name(source)
if name[0] == "/":
client.send_message(name, 1)
if (pleaseLog):
print("send " + name)
def script_load(settings):
global dispatcher
global host
sh = obs.obs_get_signal_handler()
obs.signal_handler_connect(sh, "source_activate", source_activated)
dispatcher = Dispatcher()
dispatcher.map("/*", handleOSC)
# def script_unload():
# global transport
# print(f'script_unload(settings)')
# transport.close() # Clean up serve endpoint
def script_update(settings):
global host
global client
global server
global clientPort
global serverPort
global pleaseLog
pleaseLog = obs.obs_data_get_bool(settings, "logOscOutput")
host = obs.obs_data_get_string(settings, "host")
clientPort = obs.obs_data_get_int(settings, "clientPort")
serverPort = obs.obs_data_get_int(settings, "serverPort")
# Client
client = udp_client.SimpleUDPClient(host, clientPort)
print("target set to "+host+":"+str(clientPort)+"")
# Server
print("serving in "+host+":"+str(serverPort)+"")
asyncio.run(init_main())
I'm not a programmer, so asyncio is a mess for me. I don't understand it. But I know what it does and what I'd like it to do.
When the script is loaded (in script_update(settings)), it opens the server and run loop() function. This function is a 5 seconds loop that prints some text. If I connect to the correct port from PureData and send some OSC messages, these messages arrives to OBS... but all of them drops together when the loop finishes. Meanwhile, there's nothing, OBS is blocked and nothing is in Script Log.
If I run the python-osc asyncio example code in a console, while loop is being executed, every Loop 0, Loop 1, etc, and every message arrives and all of them are printed at the right time.
How should I get this code to work? I need to open that server and run a much more code at the same time. I'm using OBS like a game engine.
Finally, I didn't use Asyncio. Instead of that, I use the Blocking Server method but inside a threading.Thread() function. For the moment, it is working right in this way.
It's important to close port when unload or reload script.
Oro, Treblig_Punisher (users from OBS Discor) helped me to achieve this.
"""
"""
# osc data
import argparse, random, time, math, threading
from pythonosc import osc_message_builder
from pythonosc import udp_client
from pythonosc import dispatcher
from pythonosc import osc_server
targetIp = "127.0.0.1"
targetPort = 10000
serverPort = 10008
client = None
server = None
#obs !
import obspython as obs
pleaseLog = False
def handleOSC(address, args, data):
print (address)
print (args)
print (data)
# defines script description
def script_description():
return '''Send and receive OSC messages'''
# defines user properties
def script_properties():
#global props
props = obs.obs_properties_create()
obs.obs_properties_add_text(props, "host", "Host IP", obs.OBS_TEXT_DEFAULT)
obs.obs_properties_add_int(props, "port", "Host port", 1, 400000, 1)
obs.obs_properties_add_bool(props, "logOscOutput", "Log OSC output")
obs.obs_properties_add_int(props, "serverPort", "Listen port", 1, 400000, 1)
return props
def script_defaults(settings):
obs.obs_data_set_default_string(settings, "host", targetIp)
obs.obs_data_set_default_int(settings, "port", targetPort)
obs.obs_data_set_default_int(settings, "serverPort", serverPort)
# Cuando se activa un source
def source_activated(cd):
global pleaseLog
source = obs.calldata_source(cd, "source")
if source is not None:
name = obs.obs_source_get_name(source)
current_scene = obs.obs_scene_from_source(obs.obs_frontend_get_current_scene())
scene_item = obs.obs_scene_find_source(current_scene, name)
boolean = obs.obs_sceneitem_visible(scene_item)
print(boolean)
if name[0] == "/":
client.send_message(name, boolean)
if (pleaseLog):
print("send " + name + boolean)
def script_load(settings):
global despachante
sh = obs.obs_get_signal_handler()
obs.signal_handler_connect(sh, "source_activate", source_activated)
despachante = dispatcher.Dispatcher()
despachante.map("/*", handleOSC)
def script_unload():
global server
server.server_close()
def script_update(settings):
global host
global port
global client
global server
global pleaseLog
pleaseLog = obs.obs_data_get_bool(settings, "logOscOutput")
host = obs.obs_data_get_string(settings, "host")
port = obs.obs_data_get_int(settings, "port")
# Client
client = udp_client.SimpleUDPClient(host, port)
print("target set to "+host+":"+str(port)+"")
# Server
serverPort = obs.obs_data_get_int(settings, "serverPort")
try:
server.server_close()
except:
print('*Server aun no creado')
# raise
server = osc_server.BlockingOSCUDPServer(("127.0.0.1", serverPort), despachante)
t = threading.Thread(target=server_th, args=([settings]))
t.daemon = True
t.start()
# Loop every 1000ms
# obs.timer_remove(funcion_loop)
# if and source_name != "":
# obs.timer_add(funcion_loop, 1000)
# Threading function
def server_th(settings):
print(f'Serving on {server.server_address}')
server.serve_forever() # Blocks forever
I am using Python3 to learn distributed programming
there are two python fileļ¼one's name is main.py, it distributes information, the other one manipulation data, and the name is worker.py.
everything goes well when I run this two file in one computer[set server address = 127.0.0.1, port = 5000]
but when i run these two files in seperate computers, they cannot connect to each other, and TimeoutError was encoutered.
I don't know why. one computer is Win10 at my home, the other is a linux cloud server which I baught.
the code works in one computer. but when I ran main.py in linux, and ran worker.py{change server to linux's ip address} in win10, then the worker.py encounter a TimeoutError
I know nothing about the linux, is there some security settings I need to open or close?
"""main.py"""
import queue
from multiprocessing.managers import BaseManager
import datetime
import time
TASK_QUEUE = queue.Queue()
RESULT_QUEUE = queue.Queue()
def get_task_queue():
"""set TASK_QUEUE as a function"""
global TASK_QUEUE
return TASK_QUEUE
def receive_result_queue():
"""set RESULT_QUEUE as a function"""
global RESULT_QUEUE
return RESULT_QUEUE
class QueueManager(BaseManager):
"""inherit BaseManager from multiprocessing.managers"""
pass
if __name__ == '__main__':
QueueManager.register('distribute_task_queue', callable=get_task_queue)
QueueManager.register('receive_result_queue', callable=receive_result_queue)
# bind port 5000, set verification code = 'abc'
MANAGER = QueueManager(address=('127.0.0.1', 5000), authkey=b'abc')
# start manager
MANAGER.start()
TASK = MANAGER.distribute_task_queue()
RESULT = MANAGER.receive_result_queue()
# put each line into manager`enter code here`
with open("C:/Users/dayia/Desktop/log.20170817") as f:
for line in f:
TASK.put(line)
# try receive result
while 1:
try:
r = RESULT.get(timeout=1)
if r[0] == r[1] and r[0] == "done":
break
else:
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),"line %s\'s length is %s" % (r[0], r[1]))
except queue.Empty:
print('result queue is empty.')
#
"""worker.py"""
import datetime
from multiprocessing.managers import BaseManager
import queue
import time
class QueueManager(BaseManager):
"""inherit BaseManager from multiprocessing.managers"""
pass
QueueManager.register('distribute_task_queue')
QueueManager.register('receive_result_queue')
server_addr = '127.0.0.1'
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), 'Connect to server %s...' % server_addr)
m = QueueManager(address=(server_addr, 5000), authkey=b'abc')
m.connect()
TASK = m.distribute_task_queue()
RESULT = m.receive_result_queue()
def parse_line(line):
return len(line)
C = 0
while not TASK.empty():
try:
n = TASK.get(timeout=1)
r = parse_line(n)
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), 'running line %s, length is %s' % (C+1, r))
C += 1
RESULT.put([r, C])
except queue.Empty:
print('task queue is empty.')
RESULT.put(["done", "done"])
enter code here
print('worker exit')
The address 127.0.0.1 very specifically refers to the same computer where the code is running (in network terms: 127.0.0.1 is the IP address of localhost) .
I have a ROUTER whose purpose is to accumulate image data from multiple DEALER clients and perform OCR on the complete image. I found that the most efficient way of handling the OCR is through the utilization of Python's multiprocessing library; the accumulated image bytes are put into a Queue for due procession in a separate Process. However, I need to ensure that when a client experiences a timeout that the Process is properly terminated and doesn't meaninglessly linger and hog resources.
In my current solution I insert each newly-connected client into a dict where the value is my ClientHandler class that possesses all image data and spawns a Thread that sets a boolean variable named "timeout" to True when 5 seconds have elapsed. Should a new message be received within that 5 second frame, bump is called & the timer is reset back to 0, otherwise I cleanup prior to thread termination and the reference is deleted from the dict in the main loop:
import threading
import time
import zmq
class ClientHandler(threading.Thread):
def __init__(self, socket):
self.elapsed = time.time()
self.timeout = False
self.socket = socket
super(ClientHandler, self).__init__()
def run(self):
while time.time() - self.elapsed < 5.0:
pass
self.timeout = True
# CLIENT TIMED OUT
# HANDLE TERMINATION AND CLEAN UP HERE
def bump(self):
self.elapsed = time.time()
def handle(self, id, header, data):
# HANDLE CLIENT DATA HERE
# ACCUMULATE IMAGE BYTES, ETC
self.socket.send_multipart([id, str(0)])
def server_task():
clients = dict()
context = zmq.Context.instance()
server = context.socket(zmq.ROUTER)
server.setsockopt(zmq.RCVTIMEO, 0)
server.bind("tcp://127.0.0.1:7777")
while True:
try:
id, header, data = server.recv_multipart()
client = clients.get(id)
if client == None:
client = clients[id] = ClientHandler(server)
client.start()
client.bump()
client.handle(id, header, data)
except zmq.Again:
for id in clients.keys():
if clients[id].timeout:
del clients[id]
context.term()
if __name__ == "__main__":
server_task()
But this entire method just doesn't feel right. Am I going about this improperly? If so, I would greatly appreciate if someone could point me in the right direction.
Figured it out myself, hoping it may be of assistance to others.
I instead have a ROUTER on an assigned port that distributes unique ports to each client, which thereafter connects to the newly-bound socket on said unique port. When a client disconnects, the port is recycled for reassignment.
import sys
import zmq
from multiprocessing import Process, Queue, Value
def server_task():
context = zmq.Context.instance()
server = context.socket(zmq.ROUTER)
server.bind("tcp://127.0.0.1:7777")
timeout_queue = Queue()
port_list = [ 1 ]
proc_list = [ ]
while True:
try:
id = server.recv_multipart()[0]
# Get an unused port from the list
# Ports from clients that have timed out are recycled here
while not timeout_queue.empty():
port_list.append(timeout_queue.get())
port = port_list.pop()
if len(port_list) == 0:
port_list.append(port + 1)
# Spawn a new worker task, binding the port to a socket
proc_running = Value("b", True)
proc_list.append(proc_running)
Process(target=worker_task, args=(proc_running, port, timeout_queue)).start()
# Send the new port to the client
server.send_multipart([id, str(7777 + port)])
except KeyboardInterrupt:
break
# Safely allow our worker processes to terminate
for proc_running in proc_list:
proc_running.value = False
context.term()
def worker_task(proc_running, port, timeout_queue):
context = zmq.Context.instance()
worker = context.socket(zmq.ROUTER)
worker.setsockopt(zmq.RCVTIMEO, 5000)
worker.bind("tcp://127.0.0.1:%d" % (7777 + port, ))
while proc_running.value:
try:
id, data = worker.recv_multipart()
worker.send_multipart([id, data])
except zmq.Again:
timeout_queue.put(port)
context.term()
break
print("Client on port %d disconnected" % (7777 + port, ))
There is a socket method for getting the IP of a given network interface:
import socket
import fcntl
import struct
def get_ip_address(ifname):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
return socket.inet_ntoa(fcntl.ioctl(
s.fileno(),
0x8915, # SIOCGIFADDR
struct.pack('256s', ifname[:15])
)[20:24])
Which returns the following:
>>> get_ip_address('lo')
'127.0.0.1'
>>> get_ip_address('eth0')
'38.113.228.130'
Is there a similar method to return the network transfer of that interface? I know I can read /proc/net/dev but I'd love a socket method.
The best way to poll ethernet interface statistics is through SNMP...
It looks like you're using linux... if so, load up your snmpd with these options... after installing snmpd, in your /etc/defaults/snmpd (make sure the line with SNMPDOPTS looks like this):
SNMPDOPTS='-Lsd -Lf /dev/null -u snmp -I -smux,usmConf,iquery,dlmod,diskio,lmSensors,hr_network,snmpEngine,system_mib,at,interface,ifTable,ipAddressTable,ifXTable,ip,cpu,tcpTable,udpTable,ipSystemStatsTable,ip,snmp_mib,tcp,icmp,udp,proc,memory,snmpNotifyTable,inetNetToMediaTable,ipSystemStatsTable,disk -Lsd -p /var/run/snmpd.pid'
You might also need to change the ro community to public See Note 1 and set your listening interfaces in /etc/snmp/snmpd.conf (if not on the loopback)...
Assuming you have a functional snmpd, at this point, you can poll ifHCInBytes and ifHCOutBytes See Note 2 for your interface(s) in question using this...
poll_bytes.py:
from SNMP import v2Manager
import time
def poll_eth0(manager=None):
# NOTE: 2nd arg to get_index should be a valid ifName value
in_bytes = manager.get_index('ifHCInOctets', 'eth0')
out_bytes = manager.get_index('ifHCOutOctets', 'eth0')
return (time.time(), int(in_bytes), int(out_bytes))
# Prep an SNMP manager object...
mgr = v2Manager('localhost')
mgr.index('ifName')
stats = list()
# Insert condition below, instead of True...
while True:
stats.append(poll_eth0(mgr))
print poll_eth0(mgr)
time.sleep(5)
SNMP.py:
from subprocess import Popen, PIPE
import re
class v2Manager(object):
def __init__(self, addr='127.0.0.1', community='public'):
self.addr = addr
self.community = community
self._index = dict()
def bulkwalk(self, oid='ifName'):
cmd = 'snmpbulkwalk -v 2c -Osq -c %s %s %s' % (self.community,
self.addr, oid)
po = Popen(cmd, shell=True, stdout=PIPE, executable='/bin/bash')
output = po.communicate()[0]
result = dict()
for line in re.split(r'\r*\n', output):
if line.strip()=="":
continue
idx, value = re.split(r'\s+', line, 1)
idx = idx.replace(oid+".", '')
result[idx] = value
return result
def bulkwalk_index(self, oid='ifOutOctets'):
result = dict()
if not (self._index==dict()):
vals = self.bulkwalk(oid=oid)
for key, val in vals.items():
idx = self._index.get(key, None)
if not (idx is None):
result[idx] = val
else:
raise ValueError, "Could not find '%s' in the index (%s)" % self.index
else:
raise ValueError, "Call the index() method before calling bulkwalk_index()"
return result
def get_index(self, oid='ifOutOctets', index=''):
# This method is horribly inefficient... improvement left as exercise for the reader...
if index:
return self.bulkwalk_index().get(index, "<unknown>")
else:
raise ValueError, "Please include an index to get"
def index(self, oid='ifName'):
self._index = self.bulkwalk(oid=oid)
END NOTES:
SNMP v2c uses clear-text authentication. If you are worried about security / someone sniffing your traffic, change your community and restrict queries to your linux machine by source ip address. The perfect world would be to modify the SNMP.py above to use SNMPv3 (which encrypts sensitive data); most people just use a non-public community and restrict snmp queries by source IP.
ifHCInOctets and ifHCOutOctets provide instantaneous values for the number of bytes transferred through the interface. If you are looking for data transfer rate, of course there will be some additional math involved.