Give sudo access to celery workers - python

I have a python program that needs to executed like this;
$ sudo python my_awesome_program.py
Now I want to run thousands of instances of this program using celery, of course with different parameters. The Problem is that while celery tries to execute my program it fails and the reason is that it doesn't have sudo access.
How can I give my celery workers the power of sudo to run this program ?
I already tried to give sudo access to my user. Changed celery service owner etc.
May be a stupid question, but I am lost.
P.S
task.py
from celery import Celery
import os
import socket
import struct
import select
import time
import logging
# Global variables
broker = "redis://%s:%s" % ("127.0.0.1", '6379')
app = Celery('process_ips', broker=broker)
logging.basicConfig(filename="/var/log/celery_ping.log", level=logging.INFO)
# From /usr/include/linux/icmp.h; your milage may vary.
ICMP_ECHO_REQUEST = 8 # Seems to be the same on Solaris.
def checksum(source_string):
"""
I'm not too confident that this is right but testing seems
to suggest that it gives the same answers as in_cksum in ping.c
"""
sum = 0
countTo = (len(source_string) / 2) * 2
count = 0
while count < countTo:
thisVal = ord(source_string[count + 1]) * \
256 + ord(source_string[count])
sum = sum + thisVal
sum = sum & 0xffffffff # Necessary?
count = count + 2
if countTo < len(source_string):
sum = sum + ord(source_string[len(source_string) - 1])
sum = sum & 0xffffffff # Necessary?
sum = (sum >> 16) + (sum & 0xffff)
sum = sum + (sum >> 16)
answer = ~sum
answer = answer & 0xffff
# Swap bytes. Bugger me if I know why.
answer = answer >> 8 | (answer << 8 & 0xff00)
return answer
def receive_one_ping(my_socket, ID, timeout):
"""
receive the ping from the socket.
"""
timeLeft = timeout
while True:
startedSelect = time.time()
whatReady = select.select([my_socket], [], [], timeLeft)
howLongInSelect = (time.time() - startedSelect)
if whatReady[0] == []: # Timeout
return
timeReceived = time.time()
recPacket, addr = my_socket.recvfrom(1024)
icmpHeader = recPacket[20:28]
type, code, checksum, packetID, sequence = struct.unpack(
"bbHHh", icmpHeader
)
if packetID == ID:
bytesInDouble = struct.calcsize("d")
timeSent = struct.unpack("d", recPacket[28:28 + bytesInDouble])[0]
return timeReceived - timeSent
timeLeft = timeLeft - howLongInSelect
if timeLeft <= 0:
return
def send_one_ping(my_socket, dest_addr, ID):
"""
Send one ping to the given >dest_addr<.
"""
dest_addr = socket.gethostbyname(dest_addr)
# Header is type (8), code (8), checksum (16), id (16), sequence (16)
my_checksum = 0
# Make a dummy heder with a 0 checksum.
header = struct.pack("bbHHh", ICMP_ECHO_REQUEST, 0, my_checksum, ID, 1)
bytesInDouble = struct.calcsize("d")
data = (192 - bytesInDouble) * "Q"
data = struct.pack("d", time.time()) + data
# Calculate the checksum on the data and the dummy header.
my_checksum = checksum(header + data)
# Now that we have the right checksum, we put that in. It's just easier
# to make up a new header than to stuff it into the dummy.
header = struct.pack(
"bbHHh", ICMP_ECHO_REQUEST, 0, socket.htons(my_checksum), ID, 1
)
packet = header + data
my_socket.sendto(packet, (dest_addr, 1)) # Don't know about the 1
def do_one(dest_addr, timeout):
"""
Returns either the delay (in seconds) or none on timeout.
"""
logging.info('Called do_one Line 105')
icmp = socket.getprotobyname("icmp")
try:
my_socket = socket.socket(socket.AF_INET, socket.SOCK_RAW, icmp)
except socket.error as xxx_todo_changeme:
(errno, msg) = xxx_todo_changeme.args
if errno == 1:
# Operation not permitted
msg = msg + (
" - Note that ICMP messages can only be sent from processes"
" running as root."
)
raise socket.error(msg)
raise # raise the original error
my_ID = os.getpid() & 0xFFFF
send_one_ping(my_socket, dest_addr, my_ID)
delay = receive_one_ping(my_socket, my_ID, timeout)
my_socket.close()
return delay
def verbose_ping(dest_addr, timeout=1, count=2):
"""
Send >count< ping to >dest_addr< with the given >timeout< and display
the result.
"""
logging.info('Messing with : %s' % dest_addr)
try:
for i in xrange(count):
logging.info('line 136')
try:
delay = do_one(dest_addr, timeout)
logging.info('line 139' + str(delay))
except socket.gaierror as e:
break
logging.info('line 142'+str(delay))
if delay is None:
pass
else:
delay = delay * 1000
logging.info('This HO is UP : %s' % dest_addr)
return dest_addr
except:
logging.info('Error in : %s' % dest_addr)
#app.task()
def process_ips(items):
logging.info('This is Items:----: %s' % items)
up_one = verbose_ping(items)
if up_one is not None:
logging.info('This one is UP: %s' % up_one)
my_awesome_program.py
from task import process_ips
if __name__ == '__main__':
for i in range(0, 256):
for j in range(1, 256):
ip = "192.168.%s.%s" % (str(i), str(j))
jobs = process_ips.delay(ip)
/etc/defaults/celeryd
# Names of nodes to start
# most will only start one node:
CELERYD_NODES="worker1"
# but you can also start multiple and configure settings
# for each in CELERYD_OPTS (see `celery multi --help` for examples).
#CELERYD_NODES="worker1 worker2 worker3"
# Absolute or relative path to the 'celery' command:
CELERY_BIN="/home/jarvis/Development/venv/bin/celery"
#CELERY_BIN="/virtualenvs/def/bin/celery"
# App instance to use
# comment out this line if you don't use an app
CELERY_APP="task"
# or fully qualified:
#CELERY_APP="proj.tasks:app"
# Where to chdir at start.
CELERYD_CHDIR="/home/jarvis/Development/pythonScrap/nmap_sub"
# Extra command-line arguments to the worker
CELERYD_OPTS="--time-limit=300 --concurrency=8 --loglevel=DEBUG"
# %N will be replaced with the first part of the nodename.
CELERYD_LOG_FILE="/var/log/celery/%N.log"
CELERYD_PID_FILE="/var/run/celery/%N.pid"
# Workers should run as an unprivileged user.
# You need to create this user manually (or you can choose
# a user/group combination that already exists, e.g. nobody).
CELERYD_USER="jarvis"
CELERYD_GROUP="jarvis"
# If enabled pid and log directories will be created if missing,
# and owned by the userid/group configured.
CELERY_CREATE_DIRS=1

You need to set your CELERYD_USER param in settings.
Have a look here: http://celery.readthedocs.org/en/latest/tutorials/daemonizing.html
If you're using supervisor, then in your supervisor conf of celery, you'd need to do this:
user=<user-you-want>
Also, under Example Configuration section, it's explicitly said to not run your workers as privileged users.

Related

My Traceroute script is giving me host name could not be determined error

Here is the CODE:
import socket
import os
import sys
import struct
import time
import select
import binascii
label = '*************{0}*************'
ICMP_ECHO_REQUEST = 8
MAX_HOPS = 30
TIMEOUT = 2.0
TRIES = 1
def checksum(string):
csum = 0
countTo = (len(string) / 2) * 2
count = 0`enter code here`
while count < countTo:
thisVal = (string[count + 1]) * 256 + (string[count])
csum = csum + thisVal
csum = csum & 0xffffffff
count = count + 2
if countTo < len(string):
csum = csum + (string[len(string) - 1])
csum = csum & 0xffffffff
csum = (csum >> 16) + (csum & 0xffff)
csum = csum + (csum >> 16)
answer = ~csum
answer = answer & 0xffff
answer = answer >> 8 | (answer << 8 & 0xff00)
return answer
def get_name_or_ip(hostip):
try:
host = socket.gethostbyaddr(hostip)
nameorip = nameorip = '{0} ({1})'.format(hostip, host[0])
return nameorip
except Exception:
nameorip = '{0} (host name could not be determined)'.format(hostip)
return nameorip
def build_packet():
myChecksum = 0
myID = os.getpid() & 0xFFFF
header = struct.pack("bbHHh", ICMP_ECHO_REQUEST, 0, myChecksum, myID, 1)
data = struct.pack("d", time.time())
myChecksum = checksum(header + data)
if sys.platform == 'darwin':
myChecksum = socket.htons(myChecksum) & 0xffff
else:
myChecksum = socket.htons(myChecksum)
header = struct.pack("bbHHh", ICMP_ECHO_REQUEST, 0, myChecksum, myID, 1)
packet = header + data
return packet
def get_route(hostname):
timeLeft = TIMEOUT
tracelist1 = [] # This is your list to use when iterating through each trace
tracelist2 = [] # This is your list to contain all traces
print(label.format(hostname))
timeLeft = TIMEOUT
for ttl in range(1, MAX_HOPS):
for tries in range(TRIES):
destAddr = socket.gethostbyname(hostname)
icmp = socket.getprotobyname("icmp")
mySocket = socket.socket(socket.AF_INET, socket.SOCK_RAW, icmp)
mySocket.setsockopt(socket.IPPROTO_IP, socket.IP_TTL,
struct.pack('I', ttl))
mySocket.settimeout(TIMEOUT)
try:
d = build_packet()
mySocket.sendto(d, (hostname, 0))
t = time.time()
startedSelect = time.time()
whatReady = select.select([mySocket], [], [], timeLeft)
howLongInSelect = (time.time() - startedSelect)
if whatReady[0] == []:
tracelist1.append(str(ttl))
tracelist1.append("* * * Request timed out.")
print(" * * * Request timed out.")
recvPacket, addr = mySocket.recvfrom(1024)
timeReceived = time.time()
timeLeft = timeLeft - howLongInSelect
if timeLeft <= 0:
tracelist1.append(str(ttl))
tracelist1.append("* * * Request timed out.")
print(" * * * Request timed out.")
except socket.timeout:
continue
else:
icmpHeaderContent = recvPacket[20:28]
type, code, checksum, packetID, sequence = struct.unpack("bbHHh", icmpHeaderContent)
printname = get_name_or_ip(addr[0])
if type == 11:
tracelist1.append(str(ttl))
tracelist1.append("* * * Request timed out.")
bytes = struct.calcsize("d")
timeSent = struct.unpack("d", recvPacket[28:28 + bytes])[0]
print(" %d rtt=%.0f ms %s" % (ttl, (timeReceived - t) * 1000, printname))
elif type == 3:
tracelist1.append(str(ttl))
tracelist1.append("* * * Request timed out.")
bytes = struct.calcsize("d")
timeSent = struct.unpack("d", recvPacket[28:28 + bytes])[0]
print(" %d rtt=%.0f ms %s" % (ttl, (timeReceived - t) * 1000, printname))
elif type == 0:
tracelist1.append("* * * Request timed out.")
bytes = struct.calcsize("d")
timeSent = struct.unpack("d", recvPacket[28:28 + bytes])[0]
print(" %d rtt=%.0f ms %s" % (ttl, (timeReceived - timeSent) * 1000, printname))
tracelist1.append(str(ttl))
tracelist2.append(tracelist1)
return tracelist2
else:
print("error")
break
#finally:
mySocket.close()
return tracelist2
if __name__ == '__main__':
get_route("google.co.il")
if len(sys.argv) < 2:
print("End)")
else:
get_route(sys.argv[1])
[1]: https://i.stack.imgur.com/TTSRd.png
Hello Good People,
I have been trying to get this school assignment to work with not avail. Basically we are required to write a traceroute script. Unfortunately, my code returns a lot of "host name could not be determined" messages. I am not sure what to do to resolve it. I am a beginner when it comes to python so please take it easy on me, I am not even sure I completely understand what I am doing from coding perspective. Any help on how to fix this will be greatly appreciated, thank you. [enter image description here][1]
Here are the instructions:
Lab 5: ICMP Traceroute Lab
In this lab you will learn how to implement a traceroute application using ICMP request and reply messages. The checksum and header making are not covered in this lab, refer to the ICMP ping lab for that purpose, the naming of most of the variables and socket is also the same.
Traceroute is a computer networking diagnostic tool which allows a user to trace the route from a host running the traceroute program to any other host in the world. Traceroute is implemented with ICMP messages. It works by sending ICMP echo (ICMP type ‘8’) messages to the same destination with increasing value of the time-to-live (TTL) field. The routers along the traceroute path return ICMP Time Exceeded (ICMP type ‘11’) when the TTL field becomes zero. The final destination sends an ICMP reply
(ICMP type ’0’) messages on receiving the ICMP echo request. The IP addresses of the routers which send replies can be extracted from the received packets. The round-trip time between the sending host and a router is determined by setting a timer at the sending host.
Your task is to develop your own Traceroute application in python using ICMP. Your application will use ICMP but, in order to keep it simple, will not exactly follow the official specification in RFC 1739.
Code
Below you will find the skeleton code for the client. You are to complete the skeleton code. The places where you need to fill in code are marked with #Fill in start and #Fill in end. Each place may require one or more lines of code.
Additional Notes
1. You do not need to be concerned about the checksum, as it is already given in the assignment skeleton code.
2. This lab requires the use of raw sockets. In some operating systems (e.g. MacOS, Windows), you may need administrator/root privileges to be able to run your Traceroute program.
3. Local testing may require you to turn your firewall or antivirus software off to allow the messages to be sent and received. However, Gradescope is not impacted by this.
4. See the end of Lab 4 ‘ICMP Pinger’ programming exercise for more information on ICMP.
What to Hand in
Use your GitHub repository to upload the complete code for the assignment. The name of the file you submit should be “solution.py”.
Testing the Pinger
Test your client by running your code to trace google.com or bing.com. Your output should return a list and meet the acceptance criteria format provided below.
Output Requirements (Acceptance Criteria)
Your code must produce the traceroute output in the format provided below for Gradescope to verify your code is working correctly.
Your trace must collect hop number, roundtrip time (rtt), host ip, and the hostname. If a hostname is not available for a host, you should provide an explicit hostname as “hostname not returnable”. Also, if a host is timing out (not responding), you must record this in your trace list item with the text “Request timed out”. Example provided below:
Example: 1 12ms 10.10.111.10 hop1.com
2 30ms 10.10.112.10 hostname not returnable
3 * Request timed out
4 5ms 10.10.110.1 target-host.com
Your get_route() function must return a nested list with trace output. That is, each trace row must be a list that includes the trace results as individual items in the list, which is also inside an overall traceroute list. Example provided below:
Example: [ [‘1’, ‘12ms’, ‘10.10.111.10’, ‘hop1.com’], [‘2’, ‘30ms’, ‘10.10.112.10’,
‘hostname not returnable’], [‘3’, ‘*’, ‘Request timed out’], [‘4’, ‘5ms’,
‘10.10.110.1’, ‘target-host.com’] ]
Note: Your output will be parsed to verify that it includes the relevant information, so if you do not provide the output of your function in a nested list, your solution will not work correctly and you will not receive points. Also, note that the example lists include all data as strings.

The response time is decreasing for some tasks

I've written a code which is supposed to receive some images and make them black & white. I'm measuring the response time for each task (response time = the time each image is received and is turned to black & white). Here is the code:
from __future__ import print_function
import signal
signal.signal(signal.SIGINT, signal.SIG_DFL)
from select import select
import socket
from struct import pack
from struct import unpack
#from collections import deque
import commands
from PIL import Image
import time
host = commands.getoutput("hostname -I")
port = 5005
backlog = 5
BUFSIZE = 4096
queueList = []
start = []
end = []
temp = []
def processP(q):
i = 0
while q:
name = q.pop(0)
col = Image.open(name)
gray = col.convert('L')
bw = gray.point(lambda x: 0 if x<128 else 255, '1')
bw.save("%d+30.jpg" % (i+1))
end.append(time.time())
#print(temp)
i = i + 1
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def read_tcp(s):
conn, addr = s.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
name = receiver.get(name_size).decode()
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
queueList.append(name)
print('name', name)
start.append(time.time())
if (name == "sample.jpg"):
print('------------ok-------------')
processP(queueList)
print("Start: ", start)
print('--------------------------')
print("End: ", end)
while start:
temp.append(end.pop(0) - start.pop(0))
print('****************************')
print("Start: ", start)
print('--------------------------')
print("End: ", end)
print("Temp: ", temp)
i = 0
while i < len(temp)-1:
if (temp[i]<temp[i+1]):
print('yes')
else:
print('No')
i = i + 1
def read_udp(s):
data,addr = s.recvfrom(1024)
print("received message:", data)
def run():
# create tcp socket
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
tcp.bind((host,port))
except socket.error as err:
print('Bind failed', err)
return
tcp.listen(1)
# create udp socket
udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
udp.bind((host,port))
print('***Socket now listening at***:', host, port)
input = [tcp,udp]
try:
while True:
inputready,outputready,exceptready = select(input,[],[])
for s in inputready:
if s == tcp:
read_tcp(s)
elif s == udp:
read_udp(s)
else:
print("unknown socket:", s)
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
raise
tcp.close()
udp.close()
if __name__ == '__main__':
run()
Now for some evaluation purposes, I send a single image many times. When I look at the response times I see that sometimes the response time of the 8th image, for example, is more than the response time of the 9th one.
So my question is that since the size and the time needed for processing each of images are the same (I'm sending a single image several times), Why is the response time for each image variable? Shouldn't the response time of the next image be longer (or at least equal) that the previous one (For example, the response time for 4th image > the response time for 3rd image)?
Your list contains the actual elapsed time it took for each image processing call. This value will be influenced by many things, including the amount of load on the system at that time.
When your program is running, it does not have exclusive access to all of the resources (cpu, ram, disk) of the system it's running on. There could be dozens, hundreds or thousands of other processes being managed by the OS vying for resources. Given this, it is highly unlikely that you would ever see even the same image processed in the exact same amount of time between two runs, when you are measuring with sub-second accuracy. The amount of time it takes can (and will) go up and down with each successive call.

Conditional Codes to return 0,1,2,3 in scapy

I would like to integrate my Scapy code with the Nagios monitoring tool and would want the results to return a 0,1,2,3.
The program works fine and acts as a passive listener to broadcast traffic and displays the IP address with the time intervals.
Can anyone help me with the code I could use to return these values, kindly find the code below.
Sorry I didn't include the code kindly find below:
import sys
import string
import datetime
import socket
from datetime import datetime
from scapy.all import *
m_iface = "eth0"
default_gw = "192.168.26.2"
COUNTER_SLOTS = 5
TIMEOUT = 20
SCREEN_REFRESH = 15
circular_counter = [0]*COUNTER_SLOTS
session_start = {}
session_stop = {}
host_names = {}
last_printed = 0
host_names ["196.168.26.254"]=u'macbook'
host_names ["192.167.26.237"]=u'testlocal'
host_names ["192.168.26.238"]=u'xp1'
host_names ["192.168.26.239"]=u'xp2'
host_names ["192.168.26.2"]=u'default gateway'
print "passive ping start"
def arp_monitor_callback(pkt):
if ARP in pkt and pkt[ARP].op in (1,2): #who-has or is-at
addr = pkt[ARP].psrc
arp_counter(addr)
return
# circular buffer for statistics, 1 slot for
if UDP in pkt and IP in pkt:
ipdata = pkt[IP]
addr = ipdata.getlayer(IP).src
arp_counter(addr)
return
def arp_counter(src):
global last_printed
tm = int(time.time())
#print (pkt.psrc)
#all_stats[src] = tm
pos = src.find("192.168.26")
if pos == -1:
# print "wrong address"
return
if src in session_stop.keys():
sess_stop = session_stop[src]
if (tm - sess_stop)/60 > TIMEOUT:
session_start[src] = tm # start a new session
session_stop[src] = tm # start a new session
else:
session_stop[src] = tm # start a new session
else: # never saw the host
session_start[src] = tm # start a new session
session_stop[src] = tm # start a new session
# print
if (tm - last_printed > SCREEN_REFRESH):
print "-----------------------------------"
last_printed = tm
i = 1
for k in sorted(session_start.keys(), cmp=lambda x, y: cmp(socket.inet_aton(x), socket.inet_aton(y))):
if k in host_names.keys():
hn = string.ljust(host_names[k], 40)
else:
hn = u'local_host_machine_ip'.ljust(40)
last_hours = (tm - session_stop[k])/3600
last_mins = ((tm - session_stop[k])/60) % 60
s_start = (datetime.fromtimestamp(int(session_start[k])).strftime('%d/%m %H:%M'))
s_stop = (datetime.fromtimestamp(int(session_stop[k])).strftime('%d/%m %H:%M'))
pos = k.find("192.168.26")
if pos != -1:
print i, k,"\t", hn,"\t",last_hours,":",last_mins,"\t"," (",s_start,"==",s_stop,")",(session_stop[k] - session_start[k])/60
i = i + 1
#sys.stdout.flush()
p = sniff(prn=arp_monitor_callback, store = 0)
Two possibilities based on the limited explanation you have provided.
Python's return statement can be used to return a Python object or in this case, an integer.
http://docs.python.org/2/reference/simple_stmts.html#return
You'll have to use it inside a function. Since you haven't posted any code, I'm not sure if this is exactly what you want.
You can also use the sys module to return an exit status when you are done running a your Scapy code.
http://docs.python.org/2/library/sys.html#sys.exit
Example using sys -
import sys
from scapy.all import *
# Scapy magic
sys.exit(0) # exit with a code of 0

WebSocket server in Python: 'module' object has no attribute 'AF_INET' [duplicate]

This question already has answers here:
Importing installed package from script with the same name raises "AttributeError: module has no attribute" or an ImportError or NameError
(2 answers)
Closed 7 months ago.
I am trying to run this simple Python WebSocket, with a couple very minor changes. I am running Python 2.4.3 because I cannot use an newer version, but I'm not sure how much that matters.
Here is the error I'm getting:
Traceback (most recent call last):
File "socket.py", line 258, in ?
server = WebSocketServer("localhost", 8000, WebSocket)
File "socket.py", line 205, in __init__
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
AttributeError: 'module' object has no attribute 'AF_INET'
And here is my code:
import time
import struct
import socket
import base64
import sys
from select import select
import re
import logging
from threading import Thread
import signal
# Simple WebSocket server implementation. Handshakes with the client then echos back everything
# that is received. Has no dependencies (doesn't require Twisted etc) and works with the RFC6455
# version of WebSockets. Tested with FireFox 16, though should work with the latest versions of
# IE, Chrome etc.
#
# rich20b#gmail.com
# Adapted from https://gist.github.com/512987 with various functions stolen from other sites, see
# below for full details.
# Constants
MAGICGUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
TEXT = 0x01
BINARY = 0x02
# WebSocket implementation
class WebSocket(object):
handshake = (
"HTTP/1.1 101 Web Socket Protocol Handshake\r\n"
"Upgrade: WebSocket\r\n"
"Connection: Upgrade\r\n"
"Sec-WebSocket-Accept: %(acceptstring)s\r\n"
"Server: TestTest\r\n"
"Access-Control-Allow-Origin: http://localhost\r\n"
"Access-Control-Allow-Credentials: true\r\n"
"\r\n"
)
# Constructor
def __init__(self, client, server):
self.client = client
self.server = server
self.handshaken = False
self.header = ""
self.data = ""
# Serve this client
def feed(self, data):
# If we haven't handshaken yet
if not self.handshaken:
logging.debug("No handshake yet")
self.header += data
if self.header.find('\r\n\r\n') != -1:
parts = self.header.split('\r\n\r\n', 1)
self.header = parts[0]
if self.dohandshake(self.header, parts[1]):
logging.info("Handshake successful")
self.handshaken = True
# We have handshaken
else:
logging.debug("Handshake is complete")
# Decode the data that we received according to section 5 of RFC6455
recv = self.decodeCharArray(data)
# Send our reply
self.sendMessage(''.join(recv).strip());
# Stolen from http://www.cs.rpi.edu/~goldsd/docs/spring2012-csci4220/websocket-py.txt
def sendMessage(self, s):
"""
Encode and send a WebSocket message
"""
# Empty message to start with
message = ""
# always send an entire message as one frame (fin)
b1 = 0x80
# in Python 2, strs are bytes and unicodes are strings
if type(s) == unicode:
b1 |= TEXT
payload = s.encode("UTF8")
elif type(s) == str:
b1 |= TEXT
payload = s
# Append 'FIN' flag to the message
message += chr(b1)
# never mask frames from the server to the client
b2 = 0
# How long is our payload?
length = len(payload)
if length < 126:
b2 |= length
message += chr(b2)
elif length < (2 ** 16) - 1:
b2 |= 126
message += chr(b2)
l = struct.pack(">H", length)
message += l
else:
l = struct.pack(">Q", length)
b2 |= 127
message += chr(b2)
message += l
# Append payload to message
message += payload
# Send to the client
self.client.send(str(message))
# Stolen from http://stackoverflow.com/questions/8125507/how-can-i-send-and-receive-websocket-messages-on-the-server-side
def decodeCharArray(self, stringStreamIn):
# Turn string values into opererable numeric byte values
byteArray = [ord(character) for character in stringStreamIn]
datalength = byteArray[1] & 127
indexFirstMask = 2
if datalength == 126:
indexFirstMask = 4
elif datalength == 127:
indexFirstMask = 10
# Extract masks
masks = [m for m in byteArray[indexFirstMask : indexFirstMask+4]]
indexFirstDataByte = indexFirstMask + 4
# List of decoded characters
decodedChars = []
i = indexFirstDataByte
j = 0
# Loop through each byte that was received
while i < len(byteArray):
# Unmask this byte and add to the decoded buffer
decodedChars.append( chr(byteArray[i] ^ masks[j % 4]) )
i += 1
j += 1
# Return the decoded string
return decodedChars
# Handshake with this client
def dohandshake(self, header, key=None):
logging.debug("Begin handshake: %s" % header)
# Get the handshake template
handshake = self.handshake
# Step through each header
for line in header.split('\r\n')[1:]:
name, value = line.split(': ', 1)
# If this is the key
if name.lower() == "sec-websocket-key":
# Append the standard GUID and get digest
combined = value + MAGICGUID
response = base64.b64encode(combined.digest())
# Replace the placeholder in the handshake response
handshake = handshake % { 'acceptstring' : response }
logging.debug("Sending handshake %s" % handshake)
self.client.send(handshake)
return True
def onmessage(self, data):
#logging.info("Got message: %s" % data)
self.send(data)
def send(self, data):
logging.info("Sent message: %s" % data)
self.client.send("\x00%s\xff" % data)
def close(self):
self.client.close()
# WebSocket server implementation
class WebSocketServer(object):
# Constructor
def __init__(self, bind, port, cls):
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind((bind, port))
self.bind = bind
self.port = port
self.cls = cls
self.connections = {}
self.listeners = [self.socket]
# Listen for requests
def listen(self, backlog=5):
self.socket.listen(backlog)
logging.info("Listening on %s" % self.port)
# Keep serving requests
self.running = True
while self.running:
# Find clients that need servicing
rList, wList, xList = select(self.listeners, [], self.listeners, 1)
for ready in rList:
if ready == self.socket:
logging.debug("New client connection")
client, address = self.socket.accept()
fileno = client.fileno()
self.listeners.append(fileno)
self.connections[fileno] = self.cls(client, self)
else:
logging.debug("Client ready for reading %s" % ready)
client = self.connections[ready].client
data = client.recv(4096)
fileno = client.fileno()
if data:
self.connections[fileno].feed(data)
else:
logging.debug("Closing client %s" % ready)
self.connections[fileno].close()
del self.connections[fileno]
self.listeners.remove(ready)
# Step though and delete broken connections
for failed in xList:
if failed == self.socket:
logging.error("Socket broke")
for fileno, conn in self.connections:
conn.close()
self.running = False
# Entry point
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(message)s")
server = WebSocketServer("localhost", 8000, WebSocket)
server_thread = Thread(target=server.listen, args=[5])
server_thread.start()
# Add SIGINT handler for killing the threads
def signal_handler(signal, frame):
logging.info("Caught Ctrl+C, shutting down...")
server.running = False
sys.exit()
signal.signal(signal.SIGINT, signal_handler)
while True:
time.sleep(100)
It appears that you've named your own file socket.py, so when you import socket, you're not getting the system library (it's just re-importing the file you're currently in - which has no AF_INET symbol). Try renaming your file something like mysocket.py.
Even after changing the file name, if you are running the python from the terminal.
(you may get the same error)
Kindly
rm -rf socket.pyc
(previously compiled bytecode)
I had the same problem, I was literally stuck here for hours, tried re installing it a million times, but found the solution.
1) Make sure the file name is not socket.py,
2) Change the directory, it will not work in the home directory due to some permission issues.
If you have by anychance saved the file as socket.py, do not copy the same file or rename it to something else, the problem will persist.
What I advice you to do is, open a new folder in a different directory, write a simple socket code which involved AF_INET. Try to run it. It should work.
Issue can be that you have a file or Cache name socket.py or socket.pyc
rm -rf socket.py
rm -rf socket.pyc
Hopefully this will resolve your import issue. Gud Luck
enter the current working directory
and remove the files named 'socket.py' and 'socket.pyc'

python Client hangs when no data to receive from server and hangs in that thread w/o letting client send

I am trying to figure out how to get my client to send and receive data 'simultaneously' and am using threads. My problem is that, depending on the way I set it up, the way here it waits for data from the server in the recieveFromServer function which is in its own thread and cannot stop it when nothing will be sent. The other way it just waits for user input, and will send to the server and then I'd call the function recieveFromServer after the client sends a message to the server which doesn't allow for fluent communication, but cannot get it to alternate automatically. How do I release the thread when the client has nothing to be sent, or there is no more to be received from the server.
It would get to long if I tried to explain everything I have tried. :)
Thanks.
The client:
from socket import *
from threading import *
import thread
import time
from struct import pack,unpack
from networklingo import *
#from exception import *
HOST = '192.168.0.105'
PORT = 21567
BUFFSIZE = 1024
ADDR = (HOST,PORT)
lock = thread.allocate_lock()
class TronClient:
def __init__(self,control=None):
self.tcpSock = socket(AF_INET,SOCK_STREAM)
#self.tcpSock.settimeout(.2)
self.recvBuff = []
def connect(self):
self.tcpSock.connect(ADDR)
self.clientUID = self.tcpSock.recv(BUFFSIZE)
print 'My clientUID is ', self.clientUID
t = Thread(target = self.receiveFromSrv())
t.setDaemon(1)
t.start()
print 'going to main loop'
self.mainLoop()
#t = Thread(target = self.mainLoop())
#t.setName('mainLoop')
#t.setDaemon(1)
#t.start()
def receiveFromSrv(self):
RECIEVING = 1
while RECIEVING:
#print 'Attempting to retrieve more data'
#lock.acquire()
#print 'Lock Aquired in recieveFromSrv'
#try:
data = self.tcpSock.recv(BUFFSIZE)
#except socket.timeout,e:
#print 'Error recieving data, ',e
#continue
#print data
if not data: continue
header = data[:6]
msgType,msgLength,clientID = unpack("hhh",header)
print msgType
print msgLength
print clientID,'\n'
msg = data[6:]
while len(msg) < msgLength:
data = self.tcpSock.recv(BUFFSIZE)
dataLen = len(data)
if dataLen <= msgLength:
msg += data
else:
remLen = msgLength-len(data) #we just need to retrieve first bit of data to complete msg
msg += data[:remLen]
self.recvBuff.append(data[remLen:])
print msg
#else:
#lock.release()
# print 'lock release in receiveFromSrv'
#time.sleep(2)
#RECIEVING = 0
def disconnect(self,data=''):
self.send(DISCONNECT_REQUEST,data)
#self.tcpSock.close()
def send(self,msgType,msg):
header = pack("hhh",msgType,len(msg),self.clientUID)
msg = header+msg
self.tcpSock.send(msg)
def mainLoop(self):
while 1:
try:
#lock.acquire()
#print 'lock aquired in mainLoop'
data = raw_input('> ')
except EOFError: # enter key hit without any data (blank line) so ignore and continue
continue
#if not data or data == '': # no valid data so just continue
# continue
if data=='exit': # client wants to disconnect, so send request to server
self.disconnect()
break
else:
self.send(TRON_CHAT,data)
#lock.release()
#print 'lock released in main loop'
#self.recieveFromSrv()
#data = self.tcpSock.recv(BUFFSIZE)
#t = Thread(target = self.receiveFromSrv())
#t.setDaemon(1)
#t.start()
if __name__ == "__main__":
cli = TronClient()
cli.connect()
#t = Thread(target = cli.connect())
#t.setName('connect')
#t.setDaemon(1)
#t.start()
The server (uses a lock when incrementing or decrementing number of clients):
from socket import *
from threading import *
import thread
from controller import *
from networklingo import *
from struct import pack,unpack
HOST = ''
PORT = 21567
BUFSIZE = 1024
ADDR = (HOST,PORT)
nclntlock = thread.allocate_lock()
class TronServer:
def __init__(self,maxConnect=4,control=None):
self.servSock = socket(AF_INET,SOCK_STREAM)
# ensure that you can restart server quickly when it terminates
self.servSock.setsockopt(SOL_SOCKET,SO_REUSEADDR,1)
self.servSock.bind(ADDR)
self.servSock.listen(maxConnect)
# keep track of number of connected clients
self.clientsConnected = 0
# give each client a unique identfier for this run of server
self.clientUID = 0
# list of all clients to cycle through for sending
self.allClients = {}
# keep track of threads
self.cliThreads = {}
#reference back to controller
self.controller = control
self.recvBuff = []
def removeClient(self,clientID,addr):
if clientID in self.allClients.keys():
self.allClients[clientID].close()
print "Disconnected from", addr
nclntlock.acquire()
self.clientsConnected -= 1
nclntlock.release()
del self.allClients[clientID]
else:
print 'ClientID is not valid'
def recieve(self,clientsock,addr):
RECIEVING = 1
# loop serving the new client
while RECIEVING: # while PLAYING???
try:
data = clientsock.recv(BUFSIZE)
except:
RECIEVING = 0
continue
# if not data: break #no data was recieved
if data != '':
print 'Recieved msg from client: ',data
header = data[:6]
msgType,msgLength,clientID = unpack("hhh",header)
print msgType
print msgLength
print clientID,'\n'
if msgType == DISCONNECT_REQUEST: #handle disconnect request
self.removeClient(clientID,addr)
else: #pass message type and message off to controller
msg = data[6:]
while len(msg) < msgLength:
data = self.tcpSock.recv(BUFSIZE)
dataLen = len(data)
if dataLen <= msgLength:
msg += data
else:
remLen = msgLength-len(data) #we just need to retrieve first bit of data to complete msg
msg += data[:remLen]
self.recvBuff.append(data[remLen:])
print msg
# echo back the same data you just recieved
#clientsock.sendall(data)
self.send(TRON_CHAT,msg,-1) #send to client 0
for k in self.allClients.keys():
if self.allClients[k] == clientsock:
self.removeClient(k,addr)
print 'deleted after hard exit from clientID ', k
#self.cliThreads[k].join()
#del self.cliThreads[k]
# then tell controller to delete player with k
break
def send(self,msgType,msg,clientID=-1):
header = pack("hhh",msgType,len(msg),clientID)
msg = header+msg
if clientID in self.allClients:
self.allClients[clientID].send(msg)
elif clientID==ALL_PLAYERS:
for k in self.allClients.keys():
self.allClients[k].send(msg)
def mainLoop(self):
global nclntlock
try:
while self.controller != None and self.controller.state == WAITING:
print 'awaiting connections'
clientsock, caddy = self.servSock.accept()
nclntlock.acquire()
self.clientsConnected += 1
nclntlock.release()
print 'Client ',self.clientUID,' connected from:',caddy
clientsock.setblocking(0)
clientsock.send(str(self.clientUID))
self.allClients[self.clientUID] = clientsock
t = Thread(target = self.recieve, args = [clientsock,caddy])
t.setName('recieve-' + str(self.clientUID))
self.cliThreads[self.clientUID] = t
self.clientUID += 1
# t.setDaemon(1)
t.start()
finally:
self.servSock.close()
if __name__ == "__main__":
serv = TronServer(control = LocalController(nPlayers = 3, fWidth = 70, fHeight = 10))
t = Thread(target = serv.mainLoop())
t.setName('mainLoop')
# t.setDaemon(1)
t.start()
I think you want to try and set the socket to non-blocking mode:
http://docs.python.org/library/socket.html#socket.socket.setblocking
Set blocking or non-blocking mode of
the socket: if flag is 0, the socket
is set to non-blocking, else to
blocking mode. Initially all sockets
are in blocking mode. In non-blocking
mode, if a recv() call doesn’t find
any data, or if a send() call can’t
immediately dispose of the data, a
error exception is raised; in blocking
mode, the calls block until they can
proceed. s.setblocking(0) is
equivalent to s.settimeout(0);
s.setblocking(1) is equivalent to
s.settimeout(None).
Also, instead of using raw sockets, have you considdered using the multiprocessing module. It is a higher-level abstraction for doing network IO. The section on Pipes & Queues is specific to sending and receiving data between a client/server.

Categories

Resources