I've written a code which is supposed to receive some images and make them black & white. I'm measuring the response time for each task (response time = the time each image is received and is turned to black & white). Here is the code:
from __future__ import print_function
import signal
signal.signal(signal.SIGINT, signal.SIG_DFL)
from select import select
import socket
from struct import pack
from struct import unpack
#from collections import deque
import commands
from PIL import Image
import time
host = commands.getoutput("hostname -I")
port = 5005
backlog = 5
BUFSIZE = 4096
queueList = []
start = []
end = []
temp = []
def processP(q):
i = 0
while q:
name = q.pop(0)
col = Image.open(name)
gray = col.convert('L')
bw = gray.point(lambda x: 0 if x<128 else 255, '1')
bw.save("%d+30.jpg" % (i+1))
end.append(time.time())
#print(temp)
i = i + 1
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def read_tcp(s):
conn, addr = s.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
name = receiver.get(name_size).decode()
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
queueList.append(name)
print('name', name)
start.append(time.time())
if (name == "sample.jpg"):
print('------------ok-------------')
processP(queueList)
print("Start: ", start)
print('--------------------------')
print("End: ", end)
while start:
temp.append(end.pop(0) - start.pop(0))
print('****************************')
print("Start: ", start)
print('--------------------------')
print("End: ", end)
print("Temp: ", temp)
i = 0
while i < len(temp)-1:
if (temp[i]<temp[i+1]):
print('yes')
else:
print('No')
i = i + 1
def read_udp(s):
data,addr = s.recvfrom(1024)
print("received message:", data)
def run():
# create tcp socket
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
tcp.bind((host,port))
except socket.error as err:
print('Bind failed', err)
return
tcp.listen(1)
# create udp socket
udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
udp.bind((host,port))
print('***Socket now listening at***:', host, port)
input = [tcp,udp]
try:
while True:
inputready,outputready,exceptready = select(input,[],[])
for s in inputready:
if s == tcp:
read_tcp(s)
elif s == udp:
read_udp(s)
else:
print("unknown socket:", s)
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
raise
tcp.close()
udp.close()
if __name__ == '__main__':
run()
Now for some evaluation purposes, I send a single image many times. When I look at the response times I see that sometimes the response time of the 8th image, for example, is more than the response time of the 9th one.
So my question is that since the size and the time needed for processing each of images are the same (I'm sending a single image several times), Why is the response time for each image variable? Shouldn't the response time of the next image be longer (or at least equal) that the previous one (For example, the response time for 4th image > the response time for 3rd image)?
Your list contains the actual elapsed time it took for each image processing call. This value will be influenced by many things, including the amount of load on the system at that time.
When your program is running, it does not have exclusive access to all of the resources (cpu, ram, disk) of the system it's running on. There could be dozens, hundreds or thousands of other processes being managed by the OS vying for resources. Given this, it is highly unlikely that you would ever see even the same image processed in the exact same amount of time between two runs, when you are measuring with sub-second accuracy. The amount of time it takes can (and will) go up and down with each successive call.
Related
***Python code:***
import serial
import pandas as pd
import time
import re
import xlrd
from msvcrt import getch
import numpy as np
i = 0
x = 0
y = 0
df = pd.read_excel(r'C:\Users\lynchfamily\Desktop\mlglovesdata.xls')
# Read COM9
# Read from COM10 as well
# Readline() only works with a timeout (IMPORTANT)
serHC = serial.Serial('COM9', 115200,timeout=.250,parity=serial.PARITY_NONE,rtscts=1) # This is the JY
serRN = serial.Serial('COM10', 115200,timeout=.250,parity=serial.PARITY_NONE,rtscts=1) # This is the silvermate
def serialin():
# Sensor lists
sensor_names = list()
sensor_values = list()
global i
# Read a certain amount of bytes from serial and then continue
# Regular expressions for finding the proper data
while i < 6:
# print(i) for debugging
global serHC
global serRN
#searchObj = re.search(r'(A\d?\d*)?(\d*)?',serHC.read(4).decode(),re.I)
#searchObjRN = re.search(r'(A\d?\d*)?(\d*)?',serRN.read(4).decode(),re.I)
# Serial data stops while in loop
# The if statements keep the false values out of the program
#if searchObj.group(1):
sensor_names.append(serHC.read(2))
#if searchObj.group(2):
sensor_values.append(serHC.read(2))
#if searchObjRN.group(1):
sensor_names.append(serRN.read(2))
#if searchObjRN.group(2):
sensor_values.append(serRN.read(2))
i = i + 1
while 1:
# Get the key from the msvcrt module
key = getch().decode('ASCII')
# If key is pressed, do something
if key:
print(key)
# Zip them together
# Final 2D list
final_2d_list = zip(sensor_names,sensor_values)
print(list(sorted(final_2d_list)))
#vals = df.Dataframe([
#df.append(vals)
#print(sorted_array_1stdim[r])
#sensor_values = [0] * 10
# Thread for reading definition
break
# Fancy recursion
sensor_values.clear()
sensor_names.clear()
i = 0
serialin()
serialin()
Arduino Code:
// The device with green colored wires
void setup() {
Serial.begin(115200);
}
void loop() {
// It won't work with the I2C while loop for some reason. Perhaps it is getting stuck up on it
Serial.print("A4");
Serial.print(analogRead(0)); // Read the local analog signal
delay(5);
Serial.print("A5");
Serial.print(analogRead(1)); // Read the local analog signal
delay(5);
Serial.print("A6");
Serial.print(analogRead(2)); // Read the local analog signal
delay(5);
Serial.print("A7");
Serial.print(analogRead(3)); // Read the local analog signal
}
I'm trying to send analog data from sensors over through bluetooth silver mate from sparkfun, and HC-06 modules to python.
I have to read the analog data at a delay of 5 seconds between each, so that the readings aren't conflicted.
The data comes through serial ports COM9 and COM10. I know that serial in python can be blocking, that's why I attempted to read it first, and then put it in a list.
I also knows that once serial has been read through, it appears to be non-blocking. When I was using serHC.readline() and serRN.readline(), I was getting something like what I'd expect to see.
However, the data in the list were not updating according to the change in the sensors. I have to admit python is not my main programming language, so that is why I'm asking for help.
I thought maybe using multiple threads might work, but I wasn't able to get the serHC and serRN variables in the main thread.
Any help will be appreciated!!
As you have discovered it is not possible to read sequentially from serial ports: a blocking read over one port implies a loss of data simultaneous sent over the other port.
Use a thread based approach.
The following sketch should be enough to get started:
import serial
import time
import re
import threading
BYTES_TO_READ = 6
# read from serial port
def read_from_serial(board, port):
print("reading from {}: port {}".format(board, port))
payload = b''
ser = serial.Serial(port, 115200,timeout=.250, parity=serial.PARITY_NONE, rtscts=1)
bytes_count = 0
while bytes_count < BYTES_TO_READ:
read_bytes = ser.read(2)
# sum number of bytes returned (not 2), you have set the timeout on serial port
# see https://pythonhosted.org/pyserial/pyserial_api.html#serial.Serial.read
bytes_count = bytes_count + len(read_bytes)
payload = payload + read_bytes
# here you have the bytes, do your logic
# ...
print("READ from {}: [{}]".format(board, payload))
return
def main():
board = {
'JY': 'COM9',
'SILVER': 'COM10'
}
threads = []
for b in board:
t = threading.Thread(target=read_from_serial, args=(b, board[b],))
threads.append(t)
t.start()
# wait for all threads termination
for t in threads:
t.join()
main()
For learning about threading: https://pymotw.com/3/threading/
Periodic read from serials
Below a sketch for reading each TIME_PERIOD seconds.
A parte the infinite while loop around the read there is a "thread" loop with a nested try/catch block
for catching serials communication problems and retrying to connect after TIME_PERIOD.
Take it just as a starting example!
import serial
import time
import re
import threading
BYTES_TO_READ = 6
TIME_PERIOD = 5
def read_message(board, port, handle):
payload = b''
bytes_count = 0
while bytes_count < BYTES_TO_READ:
read_bytes = handle.read(2)
bytes_count = bytes_count + len(read_bytes)
payload = payload + read_bytes
# here you have the bytes, do your logic
# ...
print("READ from {}: [{}]".format(board, payload))
def serial_thread(board, port):
print("reading from {}: port {}".format(board, port))
while True:
try:
handle = serial.Serial(port, 115200,timeout=.250, parity=serial.PARITY_NONE, rtscts=1)
while True:
read_message(board, port, handle)
time.sleep(TIME_PERIOD)
except Exception as e:
print("ERROR: {}".format(e))
print("retrying in {} seconds".format(TIME_PERIOD))
handle.close()
time.sleep(TIME_PERIOD)
def main():
board = {
'JY': '/dev/ttyUSB0',
'SILVER': '/dev/ttyACM0'
}
threads = []
for b in board:
t = threading.Thread(target=serial_thread, args=(b, board[b],))
threads.append(t)
t.start()
# wait for all threads termination
for t in threads:
t.join()
main()
I used python socket to make a server on my Raspberry Pi 3 (Raspbian) and a client on my laptop (Windows 10). The server stream images to the laptop at a rate of 10fps, and can reach 15fps if I push it. The problem is when I want the laptop to send back a command based on the image, the frame rate drop sharply to 3fps. The process is like this:
Pi send img => Laptop receive img => Quick process => Send command based on process result => Pi receive command, print it => Pi send img => ...
The process time for each frame does not cause this (0.02s at most for each frame), so currently I am at a loss as to why the frame rate drop so much. The image is quite large, at around 200kB and the command is only a short string at 3B. The image is in matrix form and is pickled before sending, while the command is sent as is.
Can someone please explain to me why sending back such a short command would make the frame rate drop so much? And if possible, a solution for this problem. I tried making 2 servers, one dedicated to sending images and one for receiving command, but the result is the same.
Server:
import socket
import pickle
import time
import cv2
import numpy as np
from picamera.array import PiRGBArray
from picamera import PiCamera
from SendFrameInOO import PiImageServer
def main():
# initialize the server and time stamp
ImageServer = PiImageServer()
ImageServer2 = PiImageServer()
ImageServer.openServer('192.168.0.89', 50009)
ImageServer2.openServer('192.168.0.89', 50002)
# Initialize the camera object
camera = PiCamera()
camera.resolution = (320, 240)
camera.framerate = 10 # it seems this cannot go higher than 10
# unless special measures are taken, which may
# reduce image quality
camera.exposure_mode = 'sports' #reduce blur
rawCapture = PiRGBArray(camera)
# allow the camera to warmup
time.sleep(1)
# capture frames from the camera
print('<INFO> Preparing to stream video...')
timeStart = time.time()
for frame in camera.capture_continuous(rawCapture, format="bgr",
use_video_port = True):
# grab the raw NumPy array representing the image, then initialize
# the timestamp and occupied/unoccupied text
image = frame.array
imageData = pickle.dumps(image)
ImageServer.sendFrame(imageData) # send the frame data
# receive command from laptop and print it
command = ImageServer2.recvCommand()
if command == 'BYE':
print('BYE received, ending stream session...')
break
print(command)
# clear the stream in preparation for the next one
rawCapture.truncate(0)
print('<INFO> Video stream ended')
ImageServer.closeServer()
elapsedTime = time.time() - timeStart
print('<INFO> Total elapsed time is: ', elapsedTime)
if __name__ == '__main__': main()
Client:
from SupFunctions.ServerClientFunc import PiImageClient
import time
import pickle
import cv2
def main():
# Initialize
result = 'STP'
ImageClient = PiImageClient()
ImageClient2 = PiImageClient()
# Connect to server
ImageClient.connectClient('192.168.0.89', 50009)
ImageClient2.connectClient('192.168.0.89', 50002)
print('<INFO> Connection established, preparing to receive frames...')
timeStart = time.time()
# Receiving and processing frames
while(1):
# Receive and unload a frame
imageData = ImageClient.receiveFrame()
image = pickle.loads(imageData)
cv2.imshow('Frame', image)
key = cv2.waitKey(1) & 0xFF
# Exit when q is pressed
if key == ord('q'):
ImageClient.sendCommand('BYE')
break
ImageClient2.sendCommand(result)
ImageClient.closeClient()
elapsedTime = time.time() - timeStart
print('<INFO> Total elapsed time is: ', elapsedTime)
print('Press any key to exit the program')
#cv2.imshow('Picture from server', image)
cv2.waitKey(0)
if __name__ == '__main__': main()
PiImageServer and PiImageClient:
import socket
import pickle
import time
class PiImageClient:
def __init__(self):
self.s = None
self.counter = 0
def connectClient(self, serverIP, serverPort):
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.connect((serverIP, serverPort))
def closeClient(self):
self.s.close()
def receiveOneImage(self):
imageData = b''
lenData = self.s.recv(8)
length = pickle.loads(lenData) # should be 921764 for 640x480 images
print('Data length is:', length)
while len(imageData) < length:
toRead = length-len(imageData)
imageData += self.s.recv(4096 if toRead>4096 else toRead)
#if len(imageData)%200000 <= 4096:
# print('Received: {} of {}'.format(len(imageData), length))
return imageData
def receiveFrame(self):
imageData = b''
lenData = self.s.recv(8)
length = pickle.loads(lenData)
print('Data length is:', length)
'''length = 921764 # for 640x480 images
length = 230563 # for 320x240 images'''
while len(imageData) < length:
toRead = length-len(imageData)
imageData += self.s.recv(4096 if toRead>4096 else toRead)
#if len(imageData)%200000 <= 4096:
# print('Received: {} of {}'.format(len(imageData), length))
self.counter += 1
if len(imageData) == length:
print('Successfully received frame {}'.format(self.counter))
return imageData
def sendCommand(self, command):
if len(command) != 3:
print('<WARNING> Length of command string is different from 3')
self.s.send(command.encode())
print('Command {} sent'.format(command))
class PiImageServer:
def __init__(self):
self.s = None
self.conn = None
self.addr = None
#self.currentTime = time.time()
self.currentTime = time.asctime(time.localtime(time.time()))
self.counter = 0
def openServer(self, serverIP, serverPort):
print('<INFO> Opening image server at {}:{}'.format(serverIP,
serverPort))
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.bind((serverIP, serverPort))
self.s.listen(1)
print('Waiting for client...')
self.conn, self.addr = self.s.accept()
print('Connected by', self.addr)
def closeServer(self):
print('<INFO> Closing server...')
self.conn.close()
self.s.close()
#self.currentTime = time.time()
self.currentTime = time.asctime(time.localtime(time.time()))
print('Server closed at', self.currentTime)
def sendOneImage(self, imageData):
print('<INFO> Sending only one image...')
imageDataLen = len(imageData)
lenData = pickle.dumps(imageDataLen)
print('Sending image length')
self.conn.send(lenData)
print('Sending image data')
self.conn.send(imageData)
def sendFrame(self, frameData):
self.counter += 1
print('Sending frame ', self.counter)
frameDataLen = len(frameData)
lenData = pickle.dumps(frameDataLen)
self.conn.send(lenData)
self.conn.send(frameData)
def recvCommand(self):
commandData = self.conn.recv(3)
command = commandData.decode()
return command
I believe the problem is two-fold. First, you are serializing all activity: The server is sending a complete image, then instead of continuing on to send the next image (which would better fit the definition of "streaming"), it is stopping, waiting for all bytes of the previous image to make themselves across the network to the client, then for the client to receive all bytes of the image, unpickle it, send a response and for the response to then make its way across the wire to the server.
Is there a reason you need them to be in lockstep like this? If not, try to parallelize the two sides. Have your server create a separate thread to listen for commands coming back (or simply use select to determine when the command socket has something to receive).
Second, you are likely being bitten by Nagle's algorithm (https://en.wikipedia.org/wiki/Nagle%27s_algorithm), which is intended to prevent sending numerous packets with small payloads (but lots of overhead) across the network. So, your client-side kernel has gotten your three-bytes of command data and has buffered it, waiting for you to provide more data before it sends the data to the server (it will eventually send it anyway, after a delay). To change that, you would want to use the TCP_NODELAY socket option on the client side (see https://stackoverflow.com/a/31827588/1076479).
first of all thank you for your time. I have need to transfer images from one PC (Windows) to an other (Linux) and the other way arround. I used sockets and byte streams. How ever it takes 4 seconds to send one image which is insane amount of time. Am I missing some point here? Is there any library that supports fast file transfer on windows as well as on linux?
Server or sender
import numpy
import socket
import threading
import os
from idlelib.IOBinding import encoding
import Communication
import multiprocessing
import time
import cv2
import WrapDatastructures
import sys
import ImageProcessorClient
Queue = multiprocessing.Queue()
def SendFileToClient(name, sock, image, image_number):
#ImageRequest
ImageRequest = sock.recv(1024)
#Decode the Bytestring into ascii characters
ImageRequestAsStr = ImageRequest.decode('ascii')
print("Incoming Request String:",ImageRequestAsStr)
if ImageRequestAsStr == "FILEFLAG":
#Cascade to convert grayscale image to byte array and send it to Image Processor Client
gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
#Send size of the image to processor client
memoryConsumption = sys.getsizeof(gray_image)
print("Memory Consumtion",memoryConsumption)
sendStr = "EXISTS" + str(memoryConsumption)
#Convert the string to byte because otherwise it will not be send
sock.send((sendStr.encode(encoding)))
userResponse = sock.recv(1024)
#the Responce will be received in byte and will be converted to a string to make it checkable
userResponceStr = userResponse.decode('ascii')
if userResponceStr[:2] == 'OK':
#Get byte array
print("Convert grayscale_image to bytedata")
data = WrapDatastructures.singleChannelImgToByte(gray_image)
#bytesToSend = data #send 1024 bytes
#sock.send(bytesToSend)
sendData = 0
for i in range(0,memoryConsumption,1024):
sock.send(data[i:i+1024])
else:
print("User response not known")
else:
sendStr = "ERR"
sock.send(sendStr.encode(encoding))
sock.close()
def Main():
host = "127.0.0.1"
port = 5000
s = socket.socket()
s.bind((host,port))
s.listen(5)
#init camera
print("server started.")
imageCounterPerContainer = 0
while(True):
#Take image and put it in Q
#Call the controller to turn motor
#look for client to get request
if Queue.empty() is True:
#create client
print("create Client")
while True:
c, addr = s.accept()
print("client connected ip:< " + str(addr) +">")
imageCounterPerContainer +=1
t = threading.Thread(target = SendFileToClient, args=("rtrThread",c,cv2.imread("image_33.jpg"),imageCounterPerContainer))
t.start()
s.close()
if __name__ == "__main__":
Main()
Processor
import socket
from idlelib.IOBinding import encoding
import WrapDatastructures
import cv2
import time
def Main():
host = "127.0.0.1"
port = 5000
start = time.time()
s = socket.socket()
s.connect((host,port))
getfileRequest = "FILEFLAG"
if getfileRequest != "q":
s.send(getfileRequest.encode())
data = s.recv(1024)
dataStr = data.decode('ascii')
if dataStr[:6] == "EXISTS":
filesize = int(dataStr[6:])
print("Data is available size", filesize)
sendStr = "OK"
s.send(sendStr.encode(encoding))
#create new file new_filename and
img = b""
while True:
data = s.recv(1024)
if len(data) == 0:
break
img = b"".join([img, data])
print("Download Complete")
transferedImg = WrapDatastructures.ByteToCV2(img, (2048,2448))
cv2.imshow("transfered",transferedImg)
end = time.time()
print("Duration",end -start)
cv2.waitKey(0)
s.close()
else:
print("File does not exists")#
s.close()
s.close()
if __name__ == "__main__":
Main()
So I'm making a port scanner in python...
import socket
ip = "External IP"
s = socket.socket(2, 1) #socket.AF_INET, socket.SOCK_STREAM
def porttry(ip, port):
try:
s.connect((ip, port))
return True
except:
return None
for port in range(0, 10000):
value = porttry(ip, port)
if value == None:
print("Port not opened on %d" % port)
else:
print("Port opened on %d" % port)
break
raw_input()
But this is too slow, I want to somehow be able to some how close or break code after a period of time of not returning anything.
In addition to setting socket timeout, you can also apply multi-threading technique to turbo boost the process. It will be, at best, N times faster when you have N ports to scan.
# This script runs on Python 3
import socket, threading
def TCP_connect(ip, port_number, delay, output):
TCPsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
TCPsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
TCPsock.settimeout(delay)
try:
TCPsock.connect((ip, port_number))
output[port_number] = 'Listening'
except:
output[port_number] = ''
def scan_ports(host_ip, delay):
threads = [] # To run TCP_connect concurrently
output = {} # For printing purposes
# Spawning threads to scan ports
for i in range(10000):
t = threading.Thread(target=TCP_connect, args=(host_ip, i, delay, output))
threads.append(t)
# Starting threads
for i in range(10000):
threads[i].start()
# Locking the main thread until all threads complete
for i in range(10000):
threads[i].join()
# Printing listening ports from small to large
for i in range(10000):
if output[i] == 'Listening':
print(str(i) + ': ' + output[i])
def main():
host_ip = input("Enter host IP: ")
delay = int(input("How many seconds the socket is going to wait until timeout: "))
scan_ports(host_ip, delay)
if __name__ == "__main__":
main()
here is a quick and simple port scanner, it scans 100000 ports in 180 sec:
import threading
import socket
target = 'pythonprogramming.net'
#ip = socket.gethostbyname(target)
def portscan(port):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(0.5)#
try:
con = s.connect((target,port))
print('Port :',port,"is open.")
con.close()
except:
pass
r = 1
for x in range(1,100):
t = threading.Thread(target=portscan,kwargs={'port':r})
r += 1
t.start()
Consider setting a timeout instead of a for loop by using socket.setdefaulttimeout(timeout).
This should be a bit faster.
#-*-coding:utf8;-*-
#qpy:3
#qpy:console
import socket
import os
# This is used to set a default timeout on socket
# objects.
DEFAULT_TIMEOUT = 0.5
# This is used for checking if a call to socket.connect_ex
# was successful.
SUCCESS = 0
def check_port(*host_port, timeout=DEFAULT_TIMEOUT):
''' Try to connect to a specified host on a specified port.
If the connection takes longer then the TIMEOUT we set we assume
the host is down. If the connection is a success we can safely assume
the host is up and listing on port x. If the connection fails for any
other reason we assume the host is down and the port is closed.'''
# Create and configure the socket.
sock = socket.socket()
sock.settimeout(timeout)
# the SO_REUSEADDR flag tells the kernel to reuse a local
# socket in TIME_WAIT state, without waiting for its natural
# timeout to expire.
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Like connect(address), but return an error indicator instead
# of raising an exception for errors returned by the C-level connect()
# call (other problems, such as “host not found,” can still raise exceptions).
# The error indicator is 0 if the operation succeeded, otherwise the value of
# the errnovariable. This is useful to support, for example, asynchronous connects.
connected = sock.connect_ex(host_port) is SUCCESS
# Mark the socket closed.
# The underlying system resource (e.g. a file descriptor)
# is also closed when all file objects from makefile() are closed.
# Once that happens, all future operations on the socket object will fail.
# The remote end will receive no more data (after queued data is flushed).
sock.close()
# return True if port is open or False if port is closed.
return connected
con = check_port('www.google.com', 83)
print(con)
One can use threading.Thread and threading.Condition to synchronize port check and spawning new threads.
Script example usage:
python port_scan.py google.com 70 90
Checking 70 - 80
Checking 80 - 84
Checking 84 - 90
Found active port 80
Checking 90 - 91
Checking 91 - 94
All threads started ...
port_scan.py:
# import pdb
import socket, threading
from traceback import print_exc
class AllThreadsStarted(Exception): pass
class IPv4PortScanner(object):
def __init__(self, domain, timeout=2.0, port_range=(1024, 65535), threadcount=10):
self.domain = domain
self.timeout = timeout
self.port_range = port_range
self.threadcount = threadcount
self._lock = threading.Lock()
self._condition = threading.Condition(self._lock)
self._ports_active = []
self._ports_being_checked = []
self._next_port = self.port_range[0]
def check_port_(self, port):
"If connects then port is active"
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.settimeout(self.timeout)
try:
sock.connect((self.domain, port))
with self._lock:
self._ports_active.append(port)
print ("Found active port {}".format(port))
sock.close()
except socket.timeout, ex:
return
except:
print_exc()
# pdb.set_trace()
def check_port(self, port):
"updates self._ports_being_checked list on exit of this method"
try:
self.check_port_(port)
finally:
self._condition.acquire()
self._ports_being_checked.remove(port)
self._condition.notifyAll()
self._condition.release()
def start_another_thread(self):
if self._next_port > self.port_range[1]:
raise AllThreadsStarted()
port = self._next_port
self._next_port += 1
t = threading.Thread(target=self.check_port, args=(port,))
# update books
with self._lock:
self._ports_being_checked.append(port)
t.start()
def run(self):
try:
while True:
self._condition.acquire()
while len(self._ports_being_checked) >= self.threadcount:
# we wait for some threads to complete the task
self._condition.wait()
slots_available = self.threadcount - len(self._ports_being_checked)
self._condition.release()
print ("Checking {} - {}".format(self._next_port, self._next_port+slots_available))
for i in xrange(slots_available):
self.start_another_thread()
except AllThreadsStarted, ex:
print ("All threads started ...")
except:
print_exc()
if __name__ == "__main__":
import sys
domain = sys.argv[1]
port_s = int(sys.argv[2])
port_e = int(sys.argv[3])
scanner = IPv4PortScanner(domain=domain, port_range=(port_s, port_e))
scanner.run()
I think that this one snippet could help you : http://www.coderholic.com/python-port-scanner/
socket.setdefaulttimeout(0.5)
This will make the program faster!
socket.setdefualttimeout (time)
is used to keep trying to connect with port for perticular time...when you send request and there is timeout set for 2 seconds so it will try to connect with port for 2 seconds....if there will be no response from that port in 2 seconds....it will be count as a dead port
The following port scanner has a few constants defined at the top that you can modify as needed:
PURPOSE -- help message for the command line
PORTS -- range of ports you would like scanned
POOL_SIZE -- number of processes to scan with
TIMEOUT -- how long to wait for server connection
Feel free to adapt this according to your requirements. Maybe add some command line arguments?
#! /usr/bin/env python3
import argparse
import collections
import itertools
import multiprocessing
import operator
import socket
PURPOSE = 'Scan for open ports on a computer.'
PORTS = range(1 << 16)
POOL_SIZE = 1 << 8
TIMEOUT = 0.01
def main():
"""Get computer to scan, connect with process pool, and show open ports."""
parser = argparse.ArgumentParser(description=PURPOSE)
parser.add_argument('host', type=str, help='computer you want to scan')
host = parser.parse_args().host
with multiprocessing.Pool(POOL_SIZE, socket.setdefaulttimeout, [TIMEOUT]) \
as pool:
results = pool.imap_unordered(test, ((host, port) for port in PORTS))
servers = filter(operator.itemgetter(0), results)
numbers = map(operator.itemgetter(1), servers)
ordered = sorted(numbers)
print(f'Ports open on {host}:', *format_ports(ordered), sep='\n ')
field_names = 'family', 'socket_type', 'protocol', 'canon_name', 'address'
AddressInfo = collections.namedtuple('AddressInfo', field_names)
del field_names
def test(address):
"""Try connecting to the server and return whether or not it succeeded."""
host, port = address
for info in itertools.starmap(AddressInfo, socket.getaddrinfo(host, port)):
try:
probe = socket.socket(info.family, info.socket_type, info.protocol)
except OSError:
pass
else:
try:
probe.connect(info.address)
except OSError:
pass
else:
probe.shutdown(socket.SHUT_RDWR)
return True, port
finally:
probe.close()
return False, port
def format_ports(ports):
"""Convert port numbers into strings and show all associated services."""
if ports:
for port in ports:
try:
service = socket.getservbyport(port)
except OSError:
service = '?'
yield f'{port:<5} = {service}'
else:
yield 'None'
if __name__ == '__main__':
main()
I've just finished tinkering with Concurrent Futures on a port scanner and by God it's fast:
import concurrent.futures
import socket
def scan_port(domainip: str, port: int) -> tuple:
try:
# Use a faster socket implementation
s = socket.create_connection((domainip, port), timeout=0.5)
# Check if the connection was successful
if s:
return (port, "open")
else:
return (port, "closed")
except Exception as e:
print(f"Error scanning port {port}: {e}")
return (port, "error")
openports = {}
# Scan the ports in parallel using the faster scanning code
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = [executor.submit(scan_port, domainip, port) for port in range(1, 1024)]
for future in concurrent.futures.as_completed(futures):
status = future.result()
if status[1] == "open":
openports[status[0]] = status[1]
I am trying to figure out how to get my client to send and receive data 'simultaneously' and am using threads. My problem is that, depending on the way I set it up, the way here it waits for data from the server in the recieveFromServer function which is in its own thread and cannot stop it when nothing will be sent. The other way it just waits for user input, and will send to the server and then I'd call the function recieveFromServer after the client sends a message to the server which doesn't allow for fluent communication, but cannot get it to alternate automatically. How do I release the thread when the client has nothing to be sent, or there is no more to be received from the server.
It would get to long if I tried to explain everything I have tried. :)
Thanks.
The client:
from socket import *
from threading import *
import thread
import time
from struct import pack,unpack
from networklingo import *
#from exception import *
HOST = '192.168.0.105'
PORT = 21567
BUFFSIZE = 1024
ADDR = (HOST,PORT)
lock = thread.allocate_lock()
class TronClient:
def __init__(self,control=None):
self.tcpSock = socket(AF_INET,SOCK_STREAM)
#self.tcpSock.settimeout(.2)
self.recvBuff = []
def connect(self):
self.tcpSock.connect(ADDR)
self.clientUID = self.tcpSock.recv(BUFFSIZE)
print 'My clientUID is ', self.clientUID
t = Thread(target = self.receiveFromSrv())
t.setDaemon(1)
t.start()
print 'going to main loop'
self.mainLoop()
#t = Thread(target = self.mainLoop())
#t.setName('mainLoop')
#t.setDaemon(1)
#t.start()
def receiveFromSrv(self):
RECIEVING = 1
while RECIEVING:
#print 'Attempting to retrieve more data'
#lock.acquire()
#print 'Lock Aquired in recieveFromSrv'
#try:
data = self.tcpSock.recv(BUFFSIZE)
#except socket.timeout,e:
#print 'Error recieving data, ',e
#continue
#print data
if not data: continue
header = data[:6]
msgType,msgLength,clientID = unpack("hhh",header)
print msgType
print msgLength
print clientID,'\n'
msg = data[6:]
while len(msg) < msgLength:
data = self.tcpSock.recv(BUFFSIZE)
dataLen = len(data)
if dataLen <= msgLength:
msg += data
else:
remLen = msgLength-len(data) #we just need to retrieve first bit of data to complete msg
msg += data[:remLen]
self.recvBuff.append(data[remLen:])
print msg
#else:
#lock.release()
# print 'lock release in receiveFromSrv'
#time.sleep(2)
#RECIEVING = 0
def disconnect(self,data=''):
self.send(DISCONNECT_REQUEST,data)
#self.tcpSock.close()
def send(self,msgType,msg):
header = pack("hhh",msgType,len(msg),self.clientUID)
msg = header+msg
self.tcpSock.send(msg)
def mainLoop(self):
while 1:
try:
#lock.acquire()
#print 'lock aquired in mainLoop'
data = raw_input('> ')
except EOFError: # enter key hit without any data (blank line) so ignore and continue
continue
#if not data or data == '': # no valid data so just continue
# continue
if data=='exit': # client wants to disconnect, so send request to server
self.disconnect()
break
else:
self.send(TRON_CHAT,data)
#lock.release()
#print 'lock released in main loop'
#self.recieveFromSrv()
#data = self.tcpSock.recv(BUFFSIZE)
#t = Thread(target = self.receiveFromSrv())
#t.setDaemon(1)
#t.start()
if __name__ == "__main__":
cli = TronClient()
cli.connect()
#t = Thread(target = cli.connect())
#t.setName('connect')
#t.setDaemon(1)
#t.start()
The server (uses a lock when incrementing or decrementing number of clients):
from socket import *
from threading import *
import thread
from controller import *
from networklingo import *
from struct import pack,unpack
HOST = ''
PORT = 21567
BUFSIZE = 1024
ADDR = (HOST,PORT)
nclntlock = thread.allocate_lock()
class TronServer:
def __init__(self,maxConnect=4,control=None):
self.servSock = socket(AF_INET,SOCK_STREAM)
# ensure that you can restart server quickly when it terminates
self.servSock.setsockopt(SOL_SOCKET,SO_REUSEADDR,1)
self.servSock.bind(ADDR)
self.servSock.listen(maxConnect)
# keep track of number of connected clients
self.clientsConnected = 0
# give each client a unique identfier for this run of server
self.clientUID = 0
# list of all clients to cycle through for sending
self.allClients = {}
# keep track of threads
self.cliThreads = {}
#reference back to controller
self.controller = control
self.recvBuff = []
def removeClient(self,clientID,addr):
if clientID in self.allClients.keys():
self.allClients[clientID].close()
print "Disconnected from", addr
nclntlock.acquire()
self.clientsConnected -= 1
nclntlock.release()
del self.allClients[clientID]
else:
print 'ClientID is not valid'
def recieve(self,clientsock,addr):
RECIEVING = 1
# loop serving the new client
while RECIEVING: # while PLAYING???
try:
data = clientsock.recv(BUFSIZE)
except:
RECIEVING = 0
continue
# if not data: break #no data was recieved
if data != '':
print 'Recieved msg from client: ',data
header = data[:6]
msgType,msgLength,clientID = unpack("hhh",header)
print msgType
print msgLength
print clientID,'\n'
if msgType == DISCONNECT_REQUEST: #handle disconnect request
self.removeClient(clientID,addr)
else: #pass message type and message off to controller
msg = data[6:]
while len(msg) < msgLength:
data = self.tcpSock.recv(BUFSIZE)
dataLen = len(data)
if dataLen <= msgLength:
msg += data
else:
remLen = msgLength-len(data) #we just need to retrieve first bit of data to complete msg
msg += data[:remLen]
self.recvBuff.append(data[remLen:])
print msg
# echo back the same data you just recieved
#clientsock.sendall(data)
self.send(TRON_CHAT,msg,-1) #send to client 0
for k in self.allClients.keys():
if self.allClients[k] == clientsock:
self.removeClient(k,addr)
print 'deleted after hard exit from clientID ', k
#self.cliThreads[k].join()
#del self.cliThreads[k]
# then tell controller to delete player with k
break
def send(self,msgType,msg,clientID=-1):
header = pack("hhh",msgType,len(msg),clientID)
msg = header+msg
if clientID in self.allClients:
self.allClients[clientID].send(msg)
elif clientID==ALL_PLAYERS:
for k in self.allClients.keys():
self.allClients[k].send(msg)
def mainLoop(self):
global nclntlock
try:
while self.controller != None and self.controller.state == WAITING:
print 'awaiting connections'
clientsock, caddy = self.servSock.accept()
nclntlock.acquire()
self.clientsConnected += 1
nclntlock.release()
print 'Client ',self.clientUID,' connected from:',caddy
clientsock.setblocking(0)
clientsock.send(str(self.clientUID))
self.allClients[self.clientUID] = clientsock
t = Thread(target = self.recieve, args = [clientsock,caddy])
t.setName('recieve-' + str(self.clientUID))
self.cliThreads[self.clientUID] = t
self.clientUID += 1
# t.setDaemon(1)
t.start()
finally:
self.servSock.close()
if __name__ == "__main__":
serv = TronServer(control = LocalController(nPlayers = 3, fWidth = 70, fHeight = 10))
t = Thread(target = serv.mainLoop())
t.setName('mainLoop')
# t.setDaemon(1)
t.start()
I think you want to try and set the socket to non-blocking mode:
http://docs.python.org/library/socket.html#socket.socket.setblocking
Set blocking or non-blocking mode of
the socket: if flag is 0, the socket
is set to non-blocking, else to
blocking mode. Initially all sockets
are in blocking mode. In non-blocking
mode, if a recv() call doesn’t find
any data, or if a send() call can’t
immediately dispose of the data, a
error exception is raised; in blocking
mode, the calls block until they can
proceed. s.setblocking(0) is
equivalent to s.settimeout(0);
s.setblocking(1) is equivalent to
s.settimeout(None).
Also, instead of using raw sockets, have you considdered using the multiprocessing module. It is a higher-level abstraction for doing network IO. The section on Pipes & Queues is specific to sending and receiving data between a client/server.