I wrote a little threaded port scanner, where I paid attention to closing the sockets after I'm done with them.
Despite closing the sockets, I'm getting Too many open files errors when I run it against a large number of target hosts (~ 12'000 hosts, and 5 ports, so a total of 60'000 sockets created/closed).
My understanding is that I should only encounter this error when closing the ports. I'm not really interested in the option that requires me to modify the number of open files my system can handle (recommend in other SO posts).
Full port scanner code below.
import socket
import itertools
from concurrent.futures import ThreadPoolExecutor, as_completed
def check_port(host, port):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(1)
if sock.connect_ex((host, port)) == 0:
sock.close()
return host, port
sock.close()
return False
hosts = open('/tmp/hosts').read().splitlines()
ports = [80, 443, 8000, 8080, 8443]
with ThreadPoolExecutor(max_workers=100) as executor:
future_to_host_port = [
executor.submit(check_port, host, port)
for host, port in itertools.product(hosts, ports)
]
for future in as_completed(future_to_host_port):
try:
result = future.result()
if not result:
continue
print(result)
except socket.gaierror as e:
print(f'couldn\'t resolve')
except socket.timeout as e:
print(f'socket timeout')
except Exception as e:
print(type(e))
print(f'error {e}')
Not sure this is what's causing your specific problem, but your call to close() should be inside a finally block. From connect_ex's docs:
Like connect(address), but return an error indicator instead of raising an exception for errors returned by the C-level connect() call (other problems, such as “host not found,” can still raise exceptions).
Seems to me, if you're scanning 12,000 hosts, some of them don't exist and you leak a socket (or 5) for each of them.
Try this:
def check_port(host, port):
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.settimeout(1)
if sock.connect_ex((host, port)) == 0:
return host, port
return False
Related
I am making a script that will allow multiple clients to see live camera footage from the server script, this all works fine until one of the client scripts is closed, then a ConnectionResetError is raised, To avoid this I used a try and except block to catch the ConnectionResetError but the same error is raised every time after a connection is lost. Using just socket.recv stops the ConnectionResetError but socket.recv does not return the senders address which is required for the script to send the video stream back to the client.
Server:
host = "0.0.0.0"
port = 5000
buffer_size = 1024
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind(("", port))
listeners = list() # the addresses of the clients that want the video stream
def handle_queue(sock):
while True:
try:
message, address = sock.recvfrom(buffer_size) # block the thread until a packet arrives
print(address)
message = str(message, "utf-8") # decode the message
if message == "join":
listeners.append(address) # add the list of listeners
else:
print("unknown queue msg: ", message)
except ConnectionResetError:
print("The connection was forcefully quit")
queue_handler_thread = Thread(target=handle_queue, args=(sock,), daemon=True)
queue_handler_thread.start() # start the queue
the script then uses sock.sendto() for each address in the listeners list
Client:
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.sendto(bytes("join","utf-8"), (host, port))
while True:
data, address = sock.recvfrom(max_length) # block main thread until a packet is received
I believe what you are looking for is socket.getpeername().
This will return the remote address to which the socket is connected.
Your approach in handling the data in the wire whilst the connection is closed/lost is the right way. Handling via try/catch or using a recv() or recvfrom() method to wait for a response before closing the socket.
Since posting this i have discovered that it is infact TCP and this will not work for the project I am trying to do. Any more guidance and help would be appreciated, I cant seem to find any examples of Multiclient UDP Servers that run through one socket.
I partially resolved my system, after using the Example provided by AztCrw4282.
I was able to find a solution, I instead used the socket.accept() aproach, I am unsure as to whether this is UDP or TCP (I want UDP but with the connect system I think that is the handshake for TCP, but I am not 100% sure), but it works for now.
The client has to connect to the server then the server will accept or deny the connection, upon accepting a thread will be created for that clients connection to be managed. If any errors are thrown when interacting with that client their connection will be closed.
Server
try:
ServerSocket.bind((host, port))
except socket.error as e:
print(str(e))
print('Waiting for a Connection..')
ServerSocket.listen(5)
connections = list()
def threaded_client(connection):
connection.send(str.encode('Welcome to the Server\n'))
while True:
try:
data = str(connection.recv(2048),"utf-8") # this needs to be try catched
print("Packet Recv: ", data)
if data == "join":
print("Client Joined")
if data == "quit":
break
if not data:
break
except ConnectionResetError:
break
print("Closing a connection") # need to handle leaving the stream
connection.close()
def handle_stream():
for connection in connections:
try:
connection.send(bytes(json.dumps(frame_info) ,"utf-8"))
except:
print("Packet send failure, kicking client")
connections.remove(connection)
while True:
Client, address = ServerSocket.accept()
print('Connected to: ' + address[0] + ':' + str(address[1]))
connections.append(Client)
Thread(target=threaded_client, args=(Client, ), daemon=True).start()
ThreadCount += 1
print('Thread Number: ' + str(ThreadCount))
The only part that changes for the client is the part where it connects to the server
try:
ClientSocket.connect((host, port))
except socket.error as e:
print(str(e))
I'm trying to write my own TCP Non-Blocking Server to handle multiple long lasting socket connections rather than opening many threads to handle them.
I've written my over-complicated, hard to use syntax but have the issue forms when I'm trying to detect a closed socket.
In a normal threaded TCP Socket Server I would use detected a b'' from the socket.read(size) function, However this is not possible with a nonblocking socket as it will always return a BlockingIOError
I have also tried catching theese following events
except BrokenPipeError:
conn.abort()
except ConnectionResetError:
conn.abort()
except ConnectionAbortedError:
conn.abort()
except socket.error:
conn.abort()
(conn is a class that houses the client socket and address from socket.accept())
I'm unsure what to do, but here is a deeply simplified extract from my code:
def loop_listen(self):
while self.running == True:
cr, addr = self.server.accept()
crs = SocketHandler(self, cr, addr)
self.client_handler(crs)
self.connections.append(crs)
crs.events["open"]()
crs.cr.setblocking(0)
def loop_recv(self):
while self.running == True:
time.sleep(self.poll_time)
for conn in self.connections:
try:
data = conn.cr.recv(self.poll_size)
print(data)
if (data == b''):
conn.abort()
except BlockingIOError:
data = None
except BrokenPipeError:
conn.abort()
except ConnectionResetError:
conn.abort()
except ConnectionAbortedError:
conn.abort()
except socket.error:
conn.abort()
if (data != None):
conn.events["msg"](data)
(Both loops are separate threads)
And incase you wanted it, here is the conn class
class SocketHandler:
def __init__(self, server, cr, addr):
self.server = server
self.cr = cr
self.addr = addr
self.events = {"msg": emptyCallback, "close": "emptyCallback","open":emptyCallback}
self.cache = b""
def message(self, func):
self.events["msg"] = func
def close(self, func):
self.events["close"] = func
def open(self, func):
self.events["open"] = func
def send(self, data):
self.cr.send(data)
def abort(self):
self.cr.close()
self.events["close"]()
self.server.connections.remove(conn)
This works fine on Windows but on Ubuntu it does not call the conn.abort().
Any help would be greatly appreciated.
Thanks, Sam.
The official way to detect a closed connection on a non-blocking socket is exactly the same as blocking sockets. They return empty data from recv().
Example:
# Server
import socket
import time
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('localhost', 12345))
s.listen(1)
while True:
conn, addr = s.accept()
conn.setblocking(0)
print("New connection from " + str(addr) + ".")
while True:
try:
data = conn.recv(1024)
if not data:
break
print("Received:", data)
except BlockingIOError:
time.sleep(0.001)
print("Closed.")
# Client
import socket
import time
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('localhost', 12345))
for i in range(5):
time.sleep(0.3)
s.send(str(i).encode('utf-8'))
s.close()
There is one special case where this won't work, as described in the official docs, section When Sockets Die. It happens, when sockets don't shut down gracefully. There basically is no way for recv() to detect when a socket is dead without a graceful shutdown. It might be that this is what you are seeing.
There are multiple ways to resolve that. For one, create some kind of timeout that closes and discards a socket if it didn't receive a message for a sensible amount of time. Secondly, you could actively send messages. Detecting a dead socket is much easier for send() than for recv().
Further, this works on Linux. I didn't test it on Windows. The internal implementation of the sockets class is very platform dependent, so it might be a Windows bug.
I am writing a connector using UDP in Python 3.3
When I am sending data to the UDP port, everything works fine. The problem is that when I am not sending any data, I get an error generated by the receiving port once per minute that says "timed out". While debugging, I used the socket.gettimeout() function and it returned 'None'.
Why am I getting this timeout error? Any help would be greatly appreciated!
import socket
from EventArgs import EventArgs
import logging
class UDPServer(object):
"""description of class"""
def __init__(self, onMessageReceivedEvent = '\x00'):
self.__onMessageReceivedEvent = onMessageReceivedEvent
self.__s = '\x00'
self.__r = '\x00'
def openReceivePort(self,port):
try:
self.__r = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.__r.bind(("",port))
print ("opening port: ", port)
except socket.error as e:
logging.getLogger("ConnectorLogger").critical(e)
def openBroadcastPort(self):
try:
self.__s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.__s.bind(("",2101))
print ("opening port: ", 2101)
except socket.error as e:
logging.getLogger("ConnectorLogger").critical(e)
def closePorts():
if self.__r != '\x00':
self.__r.close()
if self.__s != '\x00':
self.__s.close()
def getUDPData(self):
try:
data, addr = self.__r.recvfrom(1024)
if self.__onMessageReceivedEvent != '\x00':
eventArgs = EventArgs()
eventArgs.Addr = addr
eventArgs.Data = data
self.__onMessageReceivedEvent.fire(self, eventArgs)
except socket.error as e:
logging.getLogger("ConnectorLogger").critical(e)
def send(self,ipAddress,port,message):
try:
self.__s.sendto(message.encode(),(ipAddress,23456))
except socket.error as e:
logging.getLogger("ConnectorLogger").critical(e)
I figured out the answer to my own problem. I was using the default configuration for socket.setblocking which is 0 (non-blocking). The documentation says that using this configuration is the equivalent of using a settimeout value of 0. If I use a blocking socket, it is the equivalent of using a settimeout value of 'None'. Once I changed to a blocking socket I no longer saw this error.
socket.setblocking(flag)-Set blocking or non-blocking mode of the socket: if flag is 0, the socket is set to non- blocking, else to blocking mode. Initially all sockets are in blocking mode. In non-blocking mode, if a recv() call doesn’t find any data, or if a send() call can’t immediately dispose of the data, a error exception is raised; in blocking mode, the calls block until they can proceed. s.setblocking(0) is equivalent to s.settimeout(0.0); s.setblocking(1) is equivalent to s.settimeout(None)*
I'm trying to get a small socket communication set up on my own machine for testing purposes, but I keep getting errors like "[Errno 10053] An established connection was aborted by the software in your host machine" and "[Errno 10054] An existing connection was forcibly closed by the remote host"
The code for the server is
import socket, threading, Queue
class PiConn(threading.Thread, object):
def __init__(self, input_queue, output_queue):
threading.Thread.__init__(self)
self.input_queue = input_queue
self.output_queue = output_queue
self.HOST = ''
self.PORT = 8888
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
self.s.bind((self.HOST, self.PORT))
except socket.error, msg:
print "Binding socket failed, error message: " + msg[1]
def run(self):
self.s.listen(5)
while True:
try:
#trying to accept data
conn, addr = self.s.accept()
print "Connected to", addr
data = conn.recv(4096)
self.input_queue.put(data)
except Exception as e:
print e, "when trying to accept data"
break
try:
output = self.output_queue.get(False)
self.s.sendall(output)
print "Sent", output
except Queue.Empty:
pass
except socket.error as e:
print e, "when trying to send data"
input_queue = Queue.Queue()
output_queue = Queue.Queue()
conn = PiConn(input_queue, output_queue)
conn.start()
while True:
output_queue.put("This is sent by server")
try:
print input_queue.get(False)
except Queue.Empty:
pass
The code for the client is
import socket, threading, Queue
class GUIConn(threading.Thread, object):
def __init__(self, input_queue, output_queue):
threading.Thread.__init__(self)
self.input_queue = input_queue
self.output_queue = output_queue
self.PORT = 8888
self.PI_IP = "127.0.0.1"
try:
#Creates a socket
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
except socket.error, msg:
print 'Socket creating failed, error message:' + str(msg[1])
self.s.connect((self.PI_IP, self.PORT))
def run(self):
while True:
try:
#Trying to send data
output = self.output_queue.get(False)
self.s.sendall(output)
except Queue.Empty:
pass
except socket.error as e:
print e
try:
#trying to accept data
data = self.s.recv(4096)
self.input_queue.put(data)
except Exception as e:
print e
break
input_queue = Queue.Queue()
output_queue = Queue.Queue()
conn = GUIConn(input_queue, output_queue)
conn.start()
while True:
output_queue.put("This is sent by client")
try:
print input_queue.get(False)
except Queue.Empty:
pass
To test it, I start 2 IDLE shells, run the server, and then the client.
Any clue as to what I'm doing wrong? I'm fairly new at sockets, and I've been struggling with this all day.
Thanks in advance!
Your initial problem is caused by known issues IDLE has when working with threads.
See here and here for example.
I'm not aware of any workaround. Try running your code from terminal instead.
As to the other errors you're getting, if you post them, we can try and assist.
warning, big wall of text, read all of it before commenting
there is a huge number of problem with this small amount of code
first, the most obvious is the 'busy' loops that will use up all 100% of the cpu, not only that, it will also slowly use up all the ram as well cause you set the blocking for the queue.get to be False
you could have set it to True and it would have waited until there something and once it get that, it would loop back to the top and put another one of "This is sent by client" thus solving both the busy loop and ram usage problem
while True:
output_queue.put("This is sent by client")
try:
print input_queue.get(False) # here
except Queue.Empty:
pass
second, the way you reply/send data from the server to the client isn't through the main listening socket but the socket that is return from the self.s.accept()
so self.s.sendall(output) in the server should have been conn.sendall(output)
third, in the client code, there a chance that self.output_queue.get(False) would error with Queue.Empty and thus pass using the try and except and ended up in the blocking recv
and both the server and client would both be listening and waiting for each other to send something
fourth, self.s.accept() is blocking, after one loop in the server, it would be stuck waiting for another client while the client would send the data then end up waiting for some data
lastly, about those error you said, i can't reproduce them at all, if i have to guess, i say those error are cause by your firewall or the server isn't running (fail to bind) or something else, see here: No connection could be made because the target machine actively refused it
also, you could try a different port and maybe the first two example on this site to check if there is something weird causing problem, if those example doesn't work then there is a problem with your computer, https://docs.python.org/release/2.5.2/lib/socket-example.html
How can I know if a certain port is open/closed on linux ubuntu, not a remote system, using python?
How can I list these open ports in python?
Netstat:
Is there a way to integrate netstat output with python?
You can using the socket module to simply check if a port is open or not.
It would look something like this.
import socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
result = sock.connect_ex(('127.0.0.1',80))
if result == 0:
print "Port is open"
else:
print "Port is not open"
sock.close()
If you want to use this in a more general context, you should make sure, that the socket that you open also gets closed. So the check should be more like this:
import socket
from contextlib import closing
def check_socket(host, port):
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
if sock.connect_ex((host, port)) == 0:
print("Port is open")
else:
print("Port is not open")
For me the examples above would hang if the port wasn't open. Line 4 shows use of settimeout to prevent hanging
import socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(2) #2 Second Timeout
result = sock.connect_ex(('127.0.0.1',80))
if result == 0:
print 'port OPEN'
else:
print 'port CLOSED, connect_ex returned: '+str(result)
If you only care about the local machine, you can rely on the psutil package. You can either:
Check all ports used by a specific pid:
proc = psutil.Process(pid)
print proc.connections()
Check all ports used on the local machine:
print psutil.net_connections()
It works on Windows too.
https://github.com/giampaolo/psutil
Here's a fast multi-threaded port scanner:
from time import sleep
import socket, ipaddress, threading
max_threads = 50
final = {}
def check_port(ip, port):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # TCP
#sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
socket.setdefaulttimeout(2.0) # seconds (float)
result = sock.connect_ex((ip,port))
if result == 0:
# print ("Port is open")
final[ip] = "OPEN"
else:
# print ("Port is closed/filtered")
final[ip] = "CLOSED"
sock.close()
except:
pass
port = 80
for ip in ipaddress.IPv4Network('192.168.1.0/24'):
threading.Thread(target=check_port, args=[str(ip), port]).start()
#sleep(0.1)
# limit the number of threads.
while threading.active_count() > max_threads :
sleep(1)
print(final)
Live Demo
I found multiple solutions in this post. But some solutions have a hanging issue or takeing too much time in case of the port is not opened.Try below solution :
import socket
def port_check(HOST):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(2) #Timeout in case of port not open
try:
s.connect((HOST, 22)) #Port ,Here 22 is port
return True
except:
return False
port_check("127.0.1.1")
In case when you probing TCP ports with intention to listen on it, it’s better to actually call listen. The approach with tring to connect don’t 'see' client ports of established connections, because nobody listen on its. But these ports cannot be used to listen on its.
import socket
def check_port(port, rais=True):
""" True -- it's possible to listen on this port for TCP/IPv4 or TCP/IPv6
connections. False -- otherwise.
"""
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('127.0.0.1', port))
sock.listen(5)
sock.close()
sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
sock.bind(('::1', port))
sock.listen(5)
sock.close()
except socket.error as e:
return False
if rais:
raise RuntimeError(
"The server is already running on port {0}".format(port))
return True
Building upon the psutil solution mentioned by Joe (only works for checking local ports):
import psutil
1111 in [i.laddr.port for i in psutil.net_connections()]
returns True if port 1111 currently used.
psutil is not part of python stdlib, so you'd need to pip install psutil first. It also needs python headers to be available, so you need something like python-devel
Agree with Sachin. Just one improvement, use connect_ex instead of connect, which can avoid try except
>>> def port_check(ip_port):
... s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
... s.settimeout(1)
... return s.connect_ex(ip_port) == 0
...
>>> port_check(loc)
True
>>> port_check(loc_x)
False
>>> loc
('10.3.157.24', 6443)
>>>
Just added to mrjandro's solution some improvements like automatic resource management (making sure that the opened socket also gets closed), handling timeouts, getting rid of simple connection errors / timeouts and printing out results:
import socket
from contextlib import closing
hosts = ["host1", "host2", "host3"]
port = 22
timeout_in_seconds = 2
hosts_with_opened_port = []
hosts_with_closed_port = []
hosts_with_errors = []
def check_port(host, port, timeout_in_seconds):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(timeout_in_seconds)
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
try:
result = sock.connect_ex((host, port))
if result == 0:
print("Port {} is *** OPEN *** on host: {}".format(port, host))
hosts_with_opened_port.append(host)
else:
print("Port {} is not open on host: {}".format(port, host))
hosts_with_closed_port.append(host)
except socket.gaierror:
print("Port {} check returns a network *** ERROR *** on host: {}".format(port, host))
hosts_with_errors.append(host)
for host in hosts:
check_port(host, port, timeout_in_seconds)
print("\nHosts with opened port:")
print(hosts_with_opened_port)
print("\nHosts with closed port:")
print(hosts_with_closed_port)
print("\nHosts with errors:")
print(hosts_with_errors)
Netstat tool simply parses some /proc files like /proc/net/tcp and combines it with other files contents. Yep, it's highly platform specific, but for Linux-only solution you can stick with it. Linux kernel documentation describes these files in details so you can find there how to read them.
Please also notice your question is too ambiguous because "port" could also mean serial port (/dev/ttyS* and analogs), parallel port, etc.; I've reused understanding from another answer this is network port but I'd ask you to formulate your questions more accurately.
Please check Michael answer and vote for it. It is the right way to check open ports. Netstat and other tools are not any use if you are developing services or daemons. For instance, I am crating modbus TCP server and client services for an industrial network. The services can listen to any port, but the question is whether that port is open? The program is going to be used in different places, and I cannot check them all manually, so this is what I did:
from contextlib import closing
import socket
class example:
def __init__():
self.machine_ip = socket.gethostbyname(socket.gethostname())
self.ready:bool = self.check_socket()
def check_socket(self)->bool:
result:bool = True
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
modbus_tcp_port:int = 502
if not sock.connect_ex((self.machine_ip, modbus_tcp_port)) == 0:
result = False
return result
This will find a random port in the given range:
import socket
import random
from typing import Tuple
def find_listening_port(
port_range:Tuple[int,int]=None,
host='',
socket_type='tcp',
default:int=None
) -> int:
"""Find an available listening port
Arguments:
port_range: Optional tuple of ports to randomly search, ``[min_port, max_port]``
If omitted, then randomly search between ``[6000, 65534]``
host: Host interface to search, if omitted then bind to all interfaces
socket_type: The socket type, this should be ``tcp`` or ``udp``
default: The port to try first before randomly searching the port range
Returns:
Available port for listening
"""
def _test_port(host, port, socket_protocol):
with socket.socket(socket.AF_INET, socket_protocol) as sock:
try:
sock.bind((host, port))
if socket_type == 'tcp':
sock.listen(1)
return port
except:
pass
return -1
if port_range is None:
port_range = (6000,65534)
if socket_type == 'tcp':
socket_protocol = socket.SOCK_STREAM
elif socket_type == 'udp':
socket_protocol = socket.SOCK_DGRAM
else:
raise Exception('Invalid socket_type argument, must be: tcp or udp')
searched_ports = []
if default is not None:
port = _test_port(host, default, socket_protocol)
if port != -1:
return port
searched_ports.append(default)
for _ in range(100):
port = random.randint(port_range[0], port_range[1])
if port in searched_ports:
continue
port = _test_port(host, port, socket_protocol)
if port != -1:
return port
searched_ports.append(port)
raise Exception(f'Failed to find {socket_type} listening port for host={host}')
Example usage:
# Find a TCP port,
# first check if port 80 is availble
port = find_listening_port(
port_range=(4000, 60000),
host='',
socket_type='tcp',
default=80
)
print(f'Available TCP port: {port}')