Develop Reference

Why do paramiko produces false results while using with multi-threading?

I am trying SSH to hosts (number in hundreds) using paramiko with multithreading.
Here is my code with multithreading,
import paramiko
from concurrent.futures import ThreadPoolExecutor
# Initialising paramiko SSH Client
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Fetching hosts
file = open("hosts.txt")
content = file.readlines()
hosts = [host.strip() for host in content]
def ssh(host):
try:
print("Connecting to", host)
client.connect(host,
username="SOMEUSER",
password="SOMEPASS",
timeout=1,
banner_timeout=1,
auth_timeout=1)
print("Connected to", host)
# Need to check something here...
client.close()
print(f"Connection to {host} closed.")
return True
except:
print("FAILED to connect", host)
return False
with ThreadPoolExecutor(max_workers=4) as executor:
results = list(executor.map(ssh, hosts))
for i, host in enumerate(hosts):
print(host, "=>", results[i])
Q1: I am getting false results compared to the code without any multithreading. What is wrong with my multithreading here and how can I make this work?
Q2: Somehow the below exception is being raised (multiple times) and I have no idea how this exception is not getting caught?
Exception: Error reading SSH protocol banner
Traceback (most recent call last):
File "/home/tesit/.local/lib/python3.6/site-packages/paramiko/transport.py", line 2211, in _check_banner
buf = self.packetizer.readline(timeout)
File "/home/tesit/.local/lib/python3.6/site-packages/paramiko/packet.py", line 380, in readline
buf += self._read_timeout(timeout)
File "/home/tesit/.local/lib/python3.6/site-packages/paramiko/packet.py", line 619, in _read_timeout
raise EOFError()
EOFError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/tesit/.local/lib/python3.6/site-packages/paramiko/transport.py", line 2039, in run
self._check_banner()
File "/home/tesit/.local/lib/python3.6/site-packages/paramiko/transport.py", line 2216, in _check_banner
"Error reading SSH protocol banner" + str(e)
paramiko.ssh_exception.SSHException: Error reading SSH protocol banner

Moving the initialization of Paramiko SSH Client inside the function definition done the trick for me! Now code works correctly as compared to the one without multithreading.
Here is my code after fixing the problem
import paramiko
from concurrent.futures import ThreadPoolExecutor
file = open("hosts.txt")
content = file.readlines()
hosts = [host.strip() for host in content]
def ssh(host):
try:
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
print("Connecting to", host)
client.connect(host,
username="SOMEUSER",
password="SOMEPASS",
timeout=1,
banner_timeout=1,
auth_timeout=1)
print("Connected to", host)
# Need to check something here...
client.close()
print(f"Connection to {host} closed.")
return True
except:
print("FAILED to connect", host)
return False
with ThreadPoolExecutor(max_workers=4) as executor:
output = list(executor.map(ssh, hosts))
for i, host in enumerate(hosts):
print(host, "=>", output[i])
Even though now the code works well as I intended, the below exception is not getting caught!
Traceback (most recent call last):
File "/home/tesit/.local/lib/python3.6/site-packages/paramiko/transport.py", line 2211, in _check_banner
buf = self.packetizer.readline(timeout)
File "/home/tesit/.local/lib/python3.6/site-packages/paramiko/packet.py", line 380, in readline
buf += self._read_timeout(timeout)
File "/home/tesit/.local/lib/python3.6/site-packages/paramiko/packet.py", line 607, in _read_timeout
x = self.__socket.recv(128)
ConnectionResetError: [Errno 104] Connection reset by peer
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/tesit/.local/lib/python3.6/site-packages/paramiko/transport.py", line 2039, in run
self._check_banner()
File "/home/tesit/.local/lib/python3.6/site-packages/paramiko/transport.py", line 2216, in _check_banner
"Error reading SSH protocol banner" + str(e)
paramiko.ssh_exception.SSHException: Error reading SSH protocol banner[Errno 104] Connection reset by peer

Python P2P sockets error "This operation would block forever"

Hello I'm trying to create a P2P program to distribute messages throgh the clients.
My server.py:
import requests
from flask import Flask, jsonify, request
import logging
from daemon import Daemon
import time
import argparse
import sys
from transaction import Transaction
import settings
import socket
import threading
bind_ip = '139.59.26.12'
bind_port = 9998
my_node_addr = str(bind_ip) + ":" + str(bind_port)
clients = [("0.0.0.0", 9999), ("0.0.0.0", 9998)]
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind((bind_ip, bind_port))
server.listen(5) # max backlog of connections
print('Listening on {}:{}'.format(bind_ip, bind_port))
numbers = []
def broadcast_info(info):
for n in clients:
node_str_addr = str(n[0]) + ":" + str(n[1])
print("Selected: " + node_str_addr)
if node_str_addr != my_node_addr:
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect((n[0], n[1]))
client.send(info)
response = client.recv(4096)
def handle_client_connection(client_socket):
request = client_socket.recv(1024)
request_str = request.decode()
if request_str == 'print':
client_socket.send(str(numbers).encode('utf-8'))
print(numbers)
else:
client_socket.send('Added!'.encode('utf-8'))
client_socket.close()
num_rec = int(request.decode())
if num_rec not in numbers:
numbers.append(num_rec)
#broadcast_info(request)
while True:
client_sock, address = server.accept()
print('Accepted connection from {}:{}'.format(address[0], address[1]))
client_handler = threading.Thread(
target=handle_client_connection,
args=(client_sock,)
# without comma you'd get a... TypeError: handle_client_connection() argument after * must be a sequence, not _socketobject
)
client_handler.start()
And my client:
import socket
hostname, sld, tld, port = 'www', 'integralist', 'co.uk', 80
target = '{}.{}.{}'.format(hostname, sld, tld)
# create an ipv4 (AF_INET) socket object using the tcp protocol (SOCK_STREAM)
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# connect the client
# client.connect((target, port))
client.connect(('139.59.76.32', 9998))
# send some data (in this case a HTTP GET request)
client.send("2".encode('utf-8')) # Number or "print"
# receive the response data (4096 is recommended buffer size)
response = client.recv(4096)
print (response)
client.close()
Sometimes it works well. But then after a few tries this error appears:
Exception in thread Thread-11:
Traceback (most recent call last):
File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/usr/lib/python3.5/threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "server.py", line 43, in handle_client_connection
request = client_socket.recv(1024)
File "/usr/local/lib/python3.5/dist-packages/gevent/_socket3.py", line 336, in recv
self._wait(self._read_event)
File "/usr/local/lib/python3.5/dist-packages/gevent/_socket3.py", line 156, in _wait
self.hub.wait(watcher)
File "/usr/local/lib/python3.5/dist-packages/gevent/hub.py", line 651, in wait
result = waiter.get()
File "/usr/local/lib/python3.5/dist-packages/gevent/hub.py", line 898, in get
return self.hub.switch()
File "/usr/local/lib/python3.5/dist-packages/gevent/hub.py", line 630, in switch
return RawGreenlet.switch(self)
gevent.hub.LoopExit: ('This operation would block forever', <Hub at 0x7fdf21aa2af8 epoll pending=0 ref=0 fileno=9>)
I tried everything. In my local machine works fine. But when I try to do it on my Digitalocean VPS, then this happens. I've search about this error over the Internet but I can't find anything that helps me.
Do you know what can be the cause of this error?
Thanks

Python - determine if user is server or client

I'm writing a program that needs communication between two pc's. I know how to connect and send messages to each other. When the user opens the program, it should listen to connections. But if the user clicks 'connect', the program should stop listening for connections and connect to a user instead. How should I achieve that?
I've got this right now:
MAIN
#threaded
def getOnline(self):
# fires when the user opens the program
client.connect(PORT)
#threaded
def connect(self):
# when the user clicks connect
global IS_MASTER
IS_MASTER = True
print('Master:', IS_MASTER)
if IS_MASTER:
client.closeConnection()
server.connect(CLIENT_IP, PORT)
CLIENT
class Client():
def __init__(self):
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
def connect(self, port):
print('Listening to connections')
self.s.bind(('', port))
self.s.listen(1)
conn, addr = self.s.accept()
print('Connected by:', addr)
def closeConnection(self):
print('Not listening any longer')
self.s.close()
sys.exit()
SERVER
class Server():
def __init__(self):
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
def connect(self, host, port):
self.s.connect((host, port))
print('Connected to:', host)
Now when I click 'connect' I get: 'ConnectionAbortedError: [Errno 53] Software caused connection abort'.
Any help would be greatly appreciated!
EDIT
Full Traceback
Exception in thread Thread-1:
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/threading.py", line 639, in _bootstrap_inner
self.run()
File "/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/threading.py", line 596, in run
self._target(*self._args, **self._kwargs)
File "/Users/cedricgeerinckx/Dropbox/Redux/OSX/Redux.py", line 28, in wrapped_f
ret = f(*args, **kwargs)
File "/Users/cedricgeerinckx/Dropbox/Redux/OSX/Redux.py", line 204, in getOnline
client.connect(PORT)
File "/Users/cedricgeerinckx/Dropbox/Redux/OSX/Client.py", line 14, in connect
conn, addr = self.s.accept()
File "/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/socket.py", line 135, in accept
fd, addr = self._accept()

That exception is being thrown inside your getOnline method, which is run on startup. That means the exception is occurring when the connection made in that method is shut down by the connect method. You should just handle the exception inside getOnline:
#threaded
def getOnline(self):
# fires when the user opens the program
try:
client.connect(PORT)
except ConnectionAbortedError as e:
print("Connection shut down!")

Python server chat issue when run

I've been programming my chat but then I get an error which crashes the server and so then the next time I won't be able to use the same port and so I get socket.bind error when I run it again. How to fix?
Source code:
import socket
from threading import Thread
def clientHandler():
conn, addr = sock.accept()
print addr, "is Connected"
while 1:
data = conn.recv(1024)
if not data:
break
print "Received Message", repr(data)
HOST = ''
PORT = raw_input('PORT: ')
PORT = int(PORT)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind((HOST, PORT))
sock.listen(5)
print "Server is running......"
#Thread(target=clientHandler).start()
#Thread(target=clientHandler).start()
#Thread(target=clientHandler).start()
for i in range(5):
Thread(target=clientHandler).start()
sock.close()
This is what happens:
Server is running......
Exception in thread Thread-5:
Traceback (most recent call last):
File "C:\Python27\lib\threading.py", line 810, in __bootstrap_inner
self.run()
File "C:\Python27\lib\threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "C:/Users/Judicaël/Desktop/server_test.py", line 5, in clientHandler
conn, addr = sock.accept()
File "C:\Python27\lib\socket.py", line 202, in accept
sock, addr = self._sock.accept()
File "C:\Python27\lib\socket.py", line 170, in _dummy
raise error(EBADF, 'Bad file descriptor')
error: [Errno 9] Bad file descriptor
>>> ==================================================== RESTART ====================================================
>>>
Traceback (most recent call last):
File "C:/Users/Judicaël/Desktop/server_test.py", line 15, in <module>
sock.bind((HOST, PORT))
File "C:\Python27\lib\socket.py", line 224, in meth
return getattr(self._sock,name)(*args)
error: [Errno 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted
I am using Win7 64bit python 2.7.6 64bit version
and is there also a way to restore the ports which had been blocked or does it auto restore when restart comp?
*******FIXED by Daniel************
Fixed Code:
import socket, sys
from threading import Thread
def clientHandler(conn, addr):
print addr, "is Connected"
while 1:
data = conn.recv(1024)
if data == 'Admin: /server shutdown':
sock.close()
print 'quitting...'
quit()
if not data:
break
conn.sendall(data) # only sends to original sender
print "Received Message", repr(data)
HOST = ''
PORT = int(raw_input('PORT: '))
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind((HOST, PORT))
sock.listen(999999)
print "Server is running......"
#Thread(target=clientHandler).start()
#Thread(target=clientHandler).start()
#Thread(target=clientHandler).start()
for i in range(999999):
conn, addr = sock.accept()
Thread(target=clientHandler, args=(conn, addr)).start()
sock.close()
How do I now send the messages received to all clients?

The functions you have to call at server side is the following:
- create a socket
- bind it to a port
- listen
- accept
- handle the client communication
You have multiple threads for accept. Then you close the socket, so all accepts fail.
The correct way is
while True:
client, addr = sock.accept()
Thread(target=clientHandler, args=(client, addr)).start()

socket server on a thread not releasing the port after the process terminates

The following code runs a socket server on a thread. The client socket sends 'client: hello' to the server, and the server socket receives and replies 'server: world'.
import socket
import threading
def server():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('127.0.0.1', 12345))
sock.listen(1)
req, addr = sock.accept()
print req.recv(1024)
req.sendall('server: world')
def client():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(('127.0.0.1', 12345))
sock.sendall('client: hello')
print sock.recv(1024)
def main():
t = threading.Thread(target=server)
t.start()
client()
if __name__ == '__main__':
main()
It runs ok as expected the first time, but from the second time, if you do not wait for a good few seconds for the server to release the socket, and if you try this on a Linux machine or Mac (Windows do not get it somehow) you will run into this error:
Traceback (most recent call last):
File "socket_send_receive.py", line 24, in <module>
main()
File "socket_send_receive.py", line 21, in main
client()
File "socket_send_receive.py", line 14, in client
sock.connect(('127.0.0.1', 12345))
File "<string>", line 1, in connect
socket.error: [Errno 111] Connection refused
Exception in thread Thread-1:
Traceback (most recent call last):
File "/home/cxuan/python/2.6/lib/python2.6/threading.py", line 532, in __bootstrap_inner
self.run()
File "/home/cxuan/python/2.6/lib/python2.6/threading.py", line 484, in run
self.__target(*self.__args, **self.__kwargs)
File "socket_send_receive.py", line 6, in server
sock.bind(('127.0.0.1', 12345))
File "<string>", line 1, in bind
error: [Errno 98] Address already in use
I am looking for some insight into why this is happening and if it is possible to be genuinely resolved or what best practice should be adopted.
I know already using this option can be a workaround thanks to the other posts here on stackoverflow.
socket.SO_REUSEADDR

When a socket is closed, it ends up in a state called STATE_WAIT (see this diagram). While the socket is in this state, no one else can use the same address (ip-number/port pair) unless the SO_REUSEADDR option is set on the socket.
See e.g. the Wikipedia article on TCP for more information about how TCP works and the different states.