Develop Reference

python looping socket connections

Question regarding the python socket package
My goal is to create a simple network port scanner. This is the basic idea:
#List of ports to scan
ports = [20,21,22,23,25,80]
#dictionary of port statuses
port_status = {}
#Create socket:
MY_SOCK = s.socket(s.AF_INET, s.SOCK_STREAM)
#For loop to scan each port
for port in ports:
TARGET = ("192.168.12.123", port)
#If connection is successful, result should be int 0
result = MY_SOCK.connect_ex(TARGET)
#save port and status as key/pair value
port_status[port] = result
#Terminate socket
MY_SOCK.shutdown(s.SHUT_RDWR)
MY_SOCK.close()
I am having an issue that after the first successful socket connection, it doesn't restart properly for the following sockets.
With the syntax shown above, I get thrown an OSerror
if I don't do a shutdown, it makes one successful connection then doesn't connect to the following ports.
There must be something wrong with my closing of the socket.
Any advice? Many thanks

By running it line by line outside the loop, I managed to pinpoint the problem.
The issue is that if a connection fails, it throws an error because it could not terminate a failed connection
I solved it (incorporating h4z3's advice to move the socket creation into the for loop) in the following way:
for port in ports:
MY_SOCK = s.socket(s.AF_INET, s.SOCK_STREAM)
TARGET = ("192.168.12.123", port)
result = MY_SOCK.connect_ex(TARGET)
port_status[port] = result
#Only attempt to close connection if connection was successful (== 0)
if result == 0:
MY_SOCK.shutdown(s.SHUT_RDWR)
MY_SOCK.close()
Thanks for the assistance

Infinite loading on conn.recv python socket

I'm currently trying to make a socket connection between a server and multiple clients in Python.
In short, I have a function that is used to test if the clients are still online or dead.
To do so, i'm sending a white space to the client, and if no exception occures, it's all good, and if there is an exception I'm deleting the client from the client list.
Everything works fine, except when the client is switching network (for exemple I switch from a 2GHz network, to the same network in 5GHz). Then, the conn.recv is creating an infinite loading because the client is not receiving the data that was sent in conn.send. As a precision, both of the network where I was connected works well separately. The only solution that I found for now is restarting the server when such an event occurs..
Here is the part of the code that doesn't work (server):
# Check for clients availability
def check_clients(self, UUID):
invalid_conn = []
print(f"(UUID: {UUID}) Checking for dead clients")
if UUID not in self.all_connections: return False
# Delete dead clients for specific user (UUID)
self.socket.settimeout(2) # Also tried to set a timeout, doesn't work
print('setting timeout')
for i, conn in enumerate(self.all_connections[UUID]):
try:
print('sending blank cmd')
conn.send(str.encode(' '))
cwd = str(conn.recv(20480)) # This is the line that causes infinite loop
print('got cwd:',cwd) # This is what we are excepting from the client
except:
print('appening invalid conn to list:',conn)
invalid_conn.append(conn)
continue
print('removing timeout')
self.socket.settimeout(None)
for i in invalid_conn:
client_id = self.all_connections[UUID].index(i)
print(f'(UUID: {UUID}) Could not reach client on conn list, removing {self.all_addresses[UUID][client_id]}')
del self.all_connections[UUID][client_id]
del self.all_addresses[UUID][client_id]
print(f"(UUID: {UUID}) Dead clients check finished")
return
And here is the part of the client that should receive the server information:
# Receive commands from server
def receive_commands(self):
while True:
print("Listening for commands")
output_str = None
data = self.sock.recv(1024)
print('data:',data)
if len(data) > 0:
try:
cmd = data[:].decode("cp850")
# Do such and such events with cmd, this is a shortened version for demo
output_str = str(f'received: {cmd} from server', "cp850")
except Exception as e:
output_str = f"Command failed to execute: {str(e)}"
if output_str == None:
raise ConnectionError ("No response from server")
else:
self.print_output(output_str)
print(f"Input command: {data[:].decode('cp850')}")
print(f"Output -> {output_str}\n")
self.sock.close()
return
Please ask me if you need any more info, thank you for your help.

cannot establish connection after reboot

folks,
I faced an issue trying to reconnect to server after rebooting.. I saw other articles about similar issues but everything I try comes with the same error.
Goal
Automatically reconnect to a server after reboot
Script
ssh_client = SSHClient()
ssh_client.set_missing_host_key_policy(AutoAddPolicy())
ssh_client.connect(hostname=host,port=port, username=user, password=psw)
s = ssh_client.get_transport().open_session()
agent.AgentRequestHandler(s)
try:
stdin, stdout, stderr = ssh_client.exec_command(command, get_pty= True)
get_output(stdout)
channel = stdout.channel
stdin.close()
channel.shutdown_write()
stdout_chunks = []
stdout_chunks.append(channel.recv(len(channel.in_buffer)))
while not channel.closed or channel.recv_ready() or channel.recv_stderr_ready():
got_chunk = False
readq, _, _ = select.select([stdout.channel], [], [])
for c in readq:
if c.recv_ready():
stdout_chunks.append(channel.recv(len(c.in_buffer)))
got_chunk = True
if c.recv_stderr_ready():
stderr.channel.recv_stderr(len(c.in_stderr_buffer))
got_chunk = True
if not got_chunk \
and channel.exit_status_ready() \
and not channel.recv_stderr_ready() \
and not channel.recv_ready():
channel.shutdown_read()
channel.close()
break
stdout.close()
stderr.close()
except (ConnectionResetError, SSHException):
print('Connection died')
Error is cached by try catch block:
Connection died
Connection died
Connection died
Connection died
Connection died
Connection died
Connection died
Connection died
Connection died
Connection died
Connection died
Connection died
Connection died
Connection died
Connection died
Script which I running in a remote server ends with a reboot command:
/sbin/shutdown -r now

I will post this as an answer as it is too long to be explained in a comment.
Your code is still missing parts as we do not know how you call the try/except structure and what happens when the exception is caught. However, if I may make a guess from your indent is that if an exception is caught, you will repeat try/except somehow.
You seem to rely on the channel closed status in your logic but there is an underlying layer in the form of a TCP socket. When you reboot the server, your channel dies but so does the TCP layer. In your exception handling you will need to recreate this.
I would try something like this:
try:
...
...
...
stdout.close()
stderr.close()
except (...):
sleep(2) # to prevent a busyloop when your server is rebooting
try:
ssh_client.close() # Close the connection just in case it is alive
except:
pass # We do not care if it succeeds or fails
counter = 0 # optional
while True:
sleep(2) # to prevent a busyloop when your server is rebooting
counter += 1
if counter > X:
print("server permanently down, exiting")
exit (1)
try:
ssh_client.connect(hostname=host,port=port, username=user, password=psw)
s = ssh_client.get_transport().open_session()
break # We have a liftoff
except:
pass # Server not responding yet. Try again.
(I did not test the above code, just wrote it here to give the idea. There might be typos in there)
You can ignore the counter part. I generally use a counter to prevent programs from trying until the cows come home if the server is down for long term. If you want to keep trying, remove these. If you use them, just set X high enough to allow the server plenty of time to reboot and then some.
The key part is recreating your TCP connection after an error and only leaving the error handler when you have a working connection again.
We attempt to close the existing connection just in case it is still there to avoid exhausting server resources if the problem is not in the connection dropping but we do not care if it succeeds or fails. Then we recreate the connection from scratch.
This may or may not work in your case as we do not know from your code how you re-enter this after an exception - and you do not seem to be sure either based on your comments.

Python: Multithreaded socket server runs endlessly when client stops unexpectedly

I have created a multithreaded socket server to connect many clients to the server using python. If a client stops unexpectedly due to an exception, server runs nonstop. Is there a way to kill that particular thread alone in the server and the rest running
Server:
class ClientThread(Thread):
def __init__(self,ip,port):
Thread.__init__(self)
self.ip = ip
self.port = port
print("New server socket thread started for " + ip + ":" + str(port))
def run(self):
while True :
try:
message = conn.recv(2048)
dataInfo = message.decode('ascii')
print("recv:::::"+str(dataInfo)+"::")
except:
print("Unexpected error:", sys.exc_info()[0])
Thread._stop(self)
tcpServer = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcpServer.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcpServer.bind((TCP_IP, 0))
tcpServer.listen(10)
print("Port:"+ str(tcpServer.getsockname()[1]))
threads = []
while True:
print( "Waiting for connections from clients..." )
(conn, (ip,port)) = tcpServer.accept()
newthread = ClientThread(ip,port)
newthread.start()
threads.append(newthread)
for t in threads:
t.join()
Client:
def Main():
s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
s.connect((host,int(port)))
while True:
try:
message = input("Enter Command")
s.send(message.encode('ascii'))
except Exception as ex:
logging.exception("Unexpected error:")
break
s.close()

Sorry about a very, very long answer but here goes.
There are quite a many issues with your code. First of all, your client does not actually close the socket, as s.close() will never get executed. Your loop is interrupted at break and anything that follows it will be ignored. So change the order of these statements for the sake of good programming but it has nothing to do with your problem.
Your server code is wrong in quite a many ways. As it is currently written, it never exits. Your threads also do not work right. I have fixed your code so that it is a working, multithreaded server, but it still does not exit as I have no idea what would be the trigger to make it exit. But let us start from the main loop:
while True:
print( "Waiting for connections from clients..." )
(conn, (ip,port)) = tcpServer.accept()
newthread = ClientThread(conn, ip,port)
newthread.daemon = True
newthread.start()
threads.append(newthread) # Do we need this?
for t in threads:
t.join()
I have added passing of conn to your client thread, the reason of which becomes apparent in a moment. However, your while True loop never breaks, so you will never enter the for loop where you join your threads. If your server is meant to be run indefinitely, this is not a problem at all. Just remove the for loop and this part is fine. You do not need to join threads just for the sake of joining them. Joining threads only allows your program to block until a thread has finished executing.
Another addition is newthread.daemon = True. This sets your threads to daemonic, which means they will exit as soon as your main thread exits. Now your server responds to control + c even when there are active connections.
If your server is meant to be never ending, there is also no need to store threads in your main loop to threads list. This list just keeps growing as a new entry will be added every time a client connects and disconnects, and this leaks memory as you are not using the threads list for anything. I have kept it as it was there, but there still is no mechanism to exit the infinite loop.
Then let us move on to your thread. If you want to simplify the code, you can replace the run part with a function. There is no need to subclass Thread in this case, but this works so I have kept your structure:
class ClientThread(Thread):
def __init__(self,conn, ip,port):
Thread.__init__(self)
self.ip = ip
self.port = port
self.conn = conn
print("New server socket thread started for " + ip + ":" + str(port))
def run(self):
while True :
try:
message = self.conn.recv(2048)
if not message:
print("closed")
try:
self.conn.close()
except:
pass
return
try:
dataInfo = message.decode('ascii')
print("recv:::::"+str(dataInfo)+"::")
except UnicodeDecodeError:
print("non-ascii data")
continue
except socket.error:
print("Unexpected error:", sys.exc_info()[0])
try:
self.conn.close()
except:
pass
return
First of all, we store conn to self.conn. Your version used a global version of conn variable. This caused unexpected results when you had more than one connection to the server. conn is actually a new socket created for the client connection at accept, and this is unique to each thread. This is how servers differentiate between client connections. They listen to a known port, but when the server accepts the connection, accept creates another port for that particular connection and returns it. This is why we need to pass this to the thread and then read from self.conn instead of global conn.
Your server "hung" upon client connetion errors as there was no mechanism to detect this in your loop. If the client closes connection, socket.recv() does not raise an exception but returns nothing. This is the condition you need to detect. I am fairly sure you do not even need try/except here but it does not hurt - but you need to add the exception you are expecting here. In this case catching everything with undeclared except is just wrong. You have also another statement there potentially raising exceptions. If your client sends something that cannot be decoded with ascii codec, you would get UnicodeDecodeError (try this without error handling here, telnet to your server port and copypaste some Hebrew or Japanese into the connection and see what happens). If you just caught everything and treated as socket errors, you would now enter the thread ending part of the code just because you could not parse a message. Typically we just ignore "illegal" messages and carry on. I have added this. If you want to shut down the connection upon receiving a "bad" message, just add self.conn.close() and return to this exception handler as well.
Then when you really are encountering a socket error - or the client has closed the connection, you will need to close the socket and exit the thread. You will call close() on the socket - encapsulating it in try/except as you do not really care if it fails for not being there anymore.
And when you want to exit your thread, you just return from your run() loop. When you do this, your thread exits orderly. As simple as that.
Then there is yet another potential problem, if you are not only printing the messages but are parsing them and doing something with the data you receive. This I do not fix but leave this to you.
TCP sockets transmit data, not messages. When you build a communication protocol, you must not assume that when your recv returns, it will return a single message. When your recv() returns something, it can mean one of five things:
The client has closed the connection and nothing is returned
There is exactly one full message and you receive that
There is only a partial message. Either because you read the socket before the client had transmitted all data, or because the client sent more than 2048 bytes (even if your client never sends over 2048 bytes, a malicious client would definitely try this)
There are more than one messages waiting and you received them all
As 4, but the last message is partial.
Most socket programming mistakes are related to this. The programmer expects 2 to happen (as you do now) but they do not cater for 3-5. You should instead analyse what was received and act accordingly. If there seems to be less data than a full message, store it somewhere and wait for more data to appear. When more data appears, concatenate these and see if you now have a full message. And when you have parsed a full message from this buffer, inspect the buffer to see if there is more data there - the first part of the next message or even more full messages if your client is fast and server is slow. If you process a message and then wipe the buffer, you might have wiped also bytes from your next message.

Check if remote host is up in Python

How would I check if the remote host is up without having a port number? Is there any other way I could check other then using regular ping.
There is a possibility that the remote host might drop ping packets

This worked fine for me:
HOST_UP = True if os.system("ping -c 1 " + SOMEHOST) is 0 else False

A protocol-level PING is best, i.e., connecting to the server and interacting with it in a way that doesn't do real work. That's because it is the only real way to be sure that the service is up. An ICMP ECHO (a.k.a. ping) would only tell you that the other end's network interface is up, and even then might be blocked; FWIW, I have seen machines where all user processes were bricked but which could still be pinged. In these days of application servers, even getting a network connection might not be enough; what if the hosted app is down or otherwise non-functional? As I said, talking sweet-nothings to the actual service that you are interested in is the best, surest approach.

HOST_UP = True if os.system("ping -c 5 " + SOMEHOST.strip(";")) is 0 else False
to remove nasty script execution just add .strip(";")
-c 5
to increase the number of ping requests, if all pass than True
PS. Works only on Linux, on Windows always returns True

The best you can do is:
Try and connect on a known port (eg port 80 or 443 for HTTP or HTTPS); or
Ping the site. See Ping a site in Python?
Many sites block ICMP (the portocol used to ping sites) so you must know beforehand if the host in question has it enabled or not.
Connecting to a port tells you mixed information. It really depends on what you want to know. A port might be open but the site is effectively hung so you may get a false positive. A more stringent approach might involve using a HTTP library to execute a Web request against a site and see if you get back a response.
It really all depends on what you need to know.

Many firewalls are configured to drop ping packets without responding. In addition, some network adapters will respond to ICMP ping requests without input from the operating system network stack, which means the operating system might be down, but the host still responds to pings (usually you'll notice if you reboot the server, say, it'll start responding to pings some time before the OS actually comes up and other services start up).
The only way to be certain that a host is up is to actually try to connect to it via some well-known port (e.g. web server port 80).
Why do you need to know if the host is "up", maybe there's a better way to do it.

What about trying something that requires a RPC like a 'tasklist' command in conjunction with a ping?

I would use a port scanner. Original question states that you don't want to use a port. Then you need to specify which Protocol (Yes, this needs a port) you want to monitor: HTTP, VNC, SSH, etc. In case you want to monitor via ICMP you can use subprocess and control ping parameters, number of pings, timeout, size, etc.
import subprocess
try:
res = subprocess.Popen(['ping -t2 -c 4 110.10.0.254 &> /dev/null; echo $?'],shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
out, err = res.communicate()
out = out.rstrip()
err = err.rstrip()
print 'general.connectivity() Out: ' + out
print 'general.connectivity() Err: ' + err
if(out == "0"):
print 'general.connectivity() Successful'
return True
print 'general.connectivity() Failed'
return False
except Exception,e:
print 'general.connectivity() Exception'
return False
In case you want a port scanner
import socket
from functools import partial
from multiprocessing import Pool
from multiprocessing.pool import ThreadPool
from errno import ECONNREFUSED
NUM_CORES = 4
def portscan(target,port):
try:
# Create Socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socketTimeout = 5
s.settimeout(socketTimeout)
s.connect((target,port))
print('port_scanner.is_port_opened() ' + str(port) + " is opened")
return port
except socket.error as err:
if err.errno == ECONNREFUSED:
return False
# Wrapper function that calls portscanner
def scan_ports(server=None,port=None,portStart=None,portEnd=None,**kwargs):
p = Pool(NUM_CORES)
ping_host = partial(portscan, server)
if portStart and portStart:
return filter(bool, p.map(ping_host, range(portStart, portStart)))
else:
return filter(bool, p.map(ping_host, range(port, port+1)))
# Check if port is opened
def is_port_opened(server=None,port=None, **kwargs):
print('port_scanner.is_port_opened() Checking port...')
try:
# Add More proccesses in case we look in a range
pool = ThreadPool(processes=1)
try:
ports = list(scan_ports(server=server,port=int(port)))
print("port_scanner.is_port_opened() Port scanner done.")
if len(ports)!=0:
print('port_scanner.is_port_opened() ' + str(len(ports)) + " port(s) available.")
return True
else:
print('port_scanner.is_port_opened() port not opened: (' + port +')')
return False
except Exception, e:
raise
except Exception,e:
print e
raise