Develop Reference

Upload large file using multiple connections/threads to an SFTP server with Python Paramiko

I am trying to SFTP a file to a remote server in chunks using threads and the python paramiko library.
It opens a local file and sftp chunks to the remote server in different threads.
I am basically following this solution which uses the same approach to download large file over SFTP. I would like to send large files instead.
Downloading solution
However, I'm getting in write_chunks() on the line for chunk in infile.readv(chunks): in getting this error:
AttributeError: '_io.BufferedReader' object has no attribute 'readv'
Could anybody assist with this error please. I thought that infile is a file descriptor. I don't understand why it is an _io.BufferedReader object.
import threading, os, time, paramiko
import time, paramiko
MAX_RETRIES = 10
ftp_server = "server.com"
port = 22
remote_file = "/home/filecopy.bin"
local_file = "/home/file.bin"
ssh_conn = sftp_client = None
username = "none"
password = "none"
#you could make the number of threads relative to file size
NUM_THREADS = 2
MAX_RETRIES = 10
def make_filepart_path(file_path, part_number):
"""creates filepart path from filepath"""
return "%s.filepart.%s" % (file_path, part_number+1)
def write_chunks(chunks, tnum, remote_file_part, username, password, ftp_server, max_retries):
ssh_conn = sftp_client = None
for retry in range(max_retries):
try:
ssh_conn = paramiko.Transport((ftp_server, port))
ssh_conn.connect(username=username, password=password)
sftp_client = paramiko.SFTPClient.from_transport(ssh_conn)
with sftp_client.open(remote_file_part, "wb") as outfile:
with open(local_file, "rb") as infile:
for chunk in infile.readv(chunks):
outfile.write(chunk)
break
except (EOFError, paramiko.ssh_exception.SSHException, OSError) as x:
retry += 1
print("%s %s Thread %s - > retrying %s..." % (type(x), x, tnum, retry))
time.sleep(abs(retry) * 10)
finally:
if hasattr(sftp_client, "close") and callable(sftp_client.close):
sftp_client.close()
if hasattr(ssh_conn, "close") and callable(ssh_conn.close):
ssh_conn.close()
start_time = time.time()
for retry in range(MAX_RETRIES):
try:
ssh_conn = paramiko.Transport((ftp_server, port))
ssh_conn.connect(username=username, password=password)
sftp_client = paramiko.SFTPClient.from_transport(ssh_conn)
# connect to get the file's size in order to calculate chunks
#filesize = sftp_client.stat(remote_file).st_size
filesize = os.stat(local_file).st_size
sftp_client.close()
ssh_conn.close()
chunksize = pow(2, 12)
chunks = [(offset, chunksize) for offset in range(0, filesize, chunksize)]
thread_chunk_size = (len(chunks) // NUM_THREADS) + 1
# break the chunks into sub lists to hand off to threads
thread_chunks = [chunks[i:i+thread_chunk_size] for i in range(0, len(chunks) - 1, thread_chunk_size)]
threads = []
fileparts = []
for thread_num in range(len(thread_chunks)):
remote_file_part = make_filepart_path(remote_file, thread_num)
args = (thread_chunks[thread_num], thread_num, remote_file_part, username, password, ftp_server, MAX_RETRIES)
threads.append(threading.Thread(target=write_chunks, args=args))
fileparts.append(remote_file_part)
for thread in threads:
thread.start()
for thread in threads:
thread.join()
# join file parts into one file, remove fileparts
with sftp_client.open(remote_file_part, "wb") as outfile:
for filepart in fileparts:
with open(filepart, "rb") as infile:
outfile.write(infile.read())
os.remove(filepart)
break
except (EOFError, paramiko.ssh_exception.SSHException, OSError) as x:
retry += 1
print("%s %s - > retrying %s..." % (type(x), x, retry))
time.sleep(abs(retry) * 10)
finally:
if hasattr(sftp_client, "close") and callable(sftp_client.close):
sftp_client.close()
if hasattr(ssh_conn, "close") and callable(ssh_conn.close):
ssh_conn.close()
print("Loading File %s Took %d seconds " % (sftp_file, time.time() - start_time))
Stack trace:
Exception in thread Thread-4:
Traceback (most recent call last):
File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/usr/lib/python3.6/threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "simpleNNInference.py", line 210, in write_chunks
for chunk in infile.readv(chunks):
AttributeError: '_io.BufferedReader' object has no attribute 'readv'
Exception in thread Thread-3:
Traceback (most recent call last):
File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/usr/lib/python3.6/threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "simpleNNInference.py", line 210, in write_chunks
for chunk in infile.readv(chunks):
AttributeError: '_io.BufferedReader' object has no attribute 'readv'

For an example how to do a parallel multi part upload of one large file, see the following example.
Note that most SFTP servers (including OpenSSH) do not allow merging files remotely. So you have to revert to shell command for that.
import os
import threading
import paramiko
sftp_server = "example.com"
username = "username"
password = "password"
local_path = "/local/path/file.dat"
remote_path = "/remote/path/file.dat"
threads_count = 4
size = os.path.getsize(local_path)
part_size = int(size / threads_count)
def open_ssh():
ssh = paramiko.SSHClient()
ssh.connect(sftp_server, username=username, password=password)
return ssh
def upload_part(num, offset, part_size, remote_path_part):
print(f"Running thread {num}")
try:
ssh = open_ssh()
sftp = ssh.open_sftp()
with open(local_path, "rb") as fl:
fl.seek(offset)
with sftp.open(remote_path_part, "wb") as fr:
fr.set_pipelined(True)
size = 0
while size < part_size:
s = 32768
if size + s > part_size:
s = part_size - size
data = fl.read(s)
fr.write(data)
size += len(data)
if len(data) == 0:
break
except (paramiko.ssh_exception.SSHException) as x:
print(f"Thread {num} failed: {x}")
print(f"Thread {num} done")
print("Starting")
offset = 0
threads = []
part_filenames = []
for num in range(threads_count):
if num == threads_count - 1:
part_size = size - offset
remote_path_part = f"{remote_path}.{num}"
args = (num, offset, part_size, remote_path_part)
print(f"Starting thread {num} offset {offset} size {part_size} " +
f"part name {remote_path_part}")
thread = threading.Thread(target=upload_part, args=args)
threads.append(thread)
part_filenames.append(remote_path_part)
thread.start()
print(f"Started thread {num}")
offset += part_size
for num in range(len(threads)):
print(f"Waiting for thread {num}")
threads[num].join()
print("All thread done")
parts_list = " ".join(part_filenames)
merge_command =
f"rm \"{remote_path}\" 2> /dev/null ; " + \
f"for i in {parts_list} ; do cat \"$i\" >> {remote_path} && " + \
"rm \"$i\" || break ; done"
print(f"Merge command: {merge_command}");
ssh = open_ssh()
stdin, stdout, stderr = ssh.exec_command(merge_command)
print(stdout.read().decode("utf-8"))
print(stderr.read().decode("utf-8"))
I'm not sure how much is that backed up by the SFTP specification, but many SFTP servers, including OpenSSH, allow writing to the same file from multiple connections in parallel. So you can do even without merging the files – by uploading directly to the respective parts of the target file:
import os
import threading
import paramiko
sftp_server = "example.com"
username = "username"
password = "password"
local_path = "/local/path/file.dat"
remote_path = "/remote/path/file.dat"
threads_count = 4
size = os.path.getsize(local_path)
part_size = int(size / threads_count)
lock = threading.Lock()
created = False
def upload_part(num, offset, part_size):
print(f"Running thread {num}")
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(sftp_server, port=port, username=username, password=password)
sftp = ssh.open_sftp()
with open(local_path, "rb") as fl:
fl.seek(offset)
with lock:
global created
m = "r+" if created else "w"
created = True
fr = sftp.open(remote_path, m)
with fr:
fr.seek(offset)
fr.set_pipelined(True)
size = 0
while size < part_size:
s = 32768
if size + s > part_size:
s = part_size - size
data = fl.read(s)
fr.write(data)
size += len(data)
if len(data) == 0:
break
except (paramiko.ssh_exception.SSHException) as x:
print(f"Thread {num} failed: {x}")
print(f"Thread {num} done")
print("Starting")
offset = 0
threads = []
for num in range(threads_count):
if num == threads_count - 1:
part_size = size - offset
args = (num, offset, part_size)
print(f"Starting thread {num} offset {offset} size {part_size}")
thread = threading.Thread(target=upload_part, args=args)
threads.append(thread)
thread.start()
print(f"Started thread {num}")
offset += part_size
for num in range(len(threads)):
print(f"Waiting for thread {num}")
threads[num].join()
print("All thread done")

Python Script to configure (via SSH) Cisco Switches

I have a lot of cisco switches and routers in my environment. I have three sets of credentials (only one of them works with a particular device. I then have the IPs listed in a notepad (sub.txt). And the configuration in another notepad (config.txt)
The aim is to push the configurations (line by line) from config.txt to the list of list of IP via SSH. If the command is accepted by the devices, a log should be put into success.txt and if the command is not accepted for some reason, a log should be appended to fail.txt
But this does not work. Can you please help me fix?
import paramiko
import time
import sys
import logging
import socket
import pexpect
import traceback
from pexpect.popen_spawn import PopenSpawn
remote_conn_pre = paramiko.SSHClient()
remote_conn_pre.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ips = [i.strip() for i in open("sub.txt")]
user_local = "user1"
pass_local = "pass1"
user_aspac = "user2"
pass_aspac = "pass2"
user_batcca = "user3"
pass_batcca = "pass3"
g = open('config.txt', 'r+')
str = g.read()
g.close
success = open('success.txt', 'a')
fail = open('failed.txt', 'a')
paramiko.util.log_to_file("paramiko.log")
for ip in ips:
try:
remote_conn_pre.connect(ip, username=user_local, password=pass_local, timeout=4, look_for_keys=False, allow_agent=False)
#print ("SSH connection established to %s" + ip)
remote_conn = remote_conn_pre.invoke_shell()
print (ip + ' === local credential')
#remote_conn.send("show switch\n")
remote_conn.send((str))
time.sleep(2)
output = remote_conn.recv(5000)
print (output)
except paramiko.AuthenticationException:
try:
remote_conn_pre.connect(ip, username=user_aspac, password=pass_aspac, timeout=4, look_for_keys=False, allow_agent=False)
remote_conn1 = remote_conn_pre.invoke_shell()
print ip + ' === Global Credentials'
#output = remote_conn.recv(500)
#remote_conn.send("show switch")
#remote_conn.send("\n")
remote_conn1.send((str))
time.sleep(2)
output1 = remote_conn1.recv(5000)
print (output1)
except paramiko.AuthenticationException:
try:
#remote_conn_pre.connect(ip, username=user_batcca, password=pass_batcca, timeout=4, look_for_keys=False, allow_agent=False)
#remote_conn2 = remote_conn_pre.invoke_shell()
child = pexpect.popen_spawn.PopenSpawn('ssh ' + user_batcca + '#' + ip)
child.expect ('[pP]assword:')
child.sendline(pass_batcca)
print ip + ' === BATCCA Credential'
#output2 = remote_conn2.recv(5000)
for line in open('Config.txt').xreadlines():
child.sendline(line)
i = child.expect (['#', '^'])
if i==0:
success.write(ip + '\t' + line +'\n')
elif i==1:
fail.write(ip + '\t' + line +'\n')
time.sleep(5)
output2 = child.recv(5000)
print (output2)
except paramiko.AuthenticationException:
print ip + ' === Bad credentials'
remote_conn3 = remote_conn_pre.invoke_shell()
output3 = remote_conn3.recv(5000)
print (output3)
except paramiko.SSHException:
print ip + ' === Issues with ssh service'
except socket.error:
print ip + ' === Device unreachable'

client-server python and file opening error

I am starting in python and developing a client / server script with some input parameters in terminal,arguments:(in) (out) (ip) (port) (mode) according to the code below.In passive mode, it functions as a server, in active mode it functions as a client. But I do not understand the reason for the errors presented. Can someone explain to me what's going on? Thanks.
code
def Main():
mode = sys.argv[5]
i = 0 # Auxiliar variable in loop
if (mode == "passive"):
port = int(sys.argv[4])
mySocket = socket.socket()
mySocket.bind(('', port)) # Passive connection, for server
mySocket.listen(1)
conn, addr = mySocket.accept()
conn.settimeout(1) # Timeout = 1 segundo
conn.setblocking(0) # Non blockable mode
print("Connection from: " + str(addr))
print("Connected as server")
in_arq = open(sys.argv[1], 'r')
message = in_arq.read() # File to be send
out_arq = open(sys.argv[2], 'w') # File to save data received
elif (mode == "ative"):
host = sys.argv[3]
port = int(sys.argv[4])
mySocket = socket.socket()
mySocket.connect((host, port)) # Active connection, for client
mySocket.settimeout(1) # Timeout = 1 segundo
mySocket.setblocking(0) # Non blockable mode
print("Connected as client")
in_arq = open(sys.argv[1], 'r')
message = in_arq.read() # File to be send
out_arq = open(sys.argv[2], 'w') # File to save data received
else:
print("Wrong arguments.")
if __name__ == '__main__':
Main()
errors:
C:\Users\AppData\Local\Programs\Python\Python35-32\python.exe C:/Users/Desktop/emulator.py C:\Users\Desktop\textinput C:\Users\Desktop\textoutput 127.0.0.1 54000 ative
Traceback (most recent call last):
File "C:/Users/Desktop/emulator.py", line 221, in <module>
Main()
File "C:/Users/Desktop/emulator.py", line 202, in Main
in_arq = open(sys.argv[1], 'r')
FileNotFoundError: [Errno 2] No such file or directory: 'C:\\Users\\Desktop\\textinput'
Connected as client

socket.gaierror: Errno 11004 getaddrinfo failed

i am trying to create a proxy server script with python when i run it i have this arror messages please can i know what are the mistakes i've done and how to avoid them (i sow the script on a site)
how the script looks like !
import socket
from thread import *
import sys
host = ""
port = 91
def start():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host, port))
s.listen(5)
print "[+] listening ..."
while True:
try:
connection, address = s.accept()
data = connection.recv(1024)
start_new_thread(conn_string, (data, connection))
except KeyboardInterrupt:
print "\n\nclosing !"
def conn_string(data, con):
webserver = ""
portserver = 0
f_li = data.split('\n')[0]
lien = f_li.split(' ')[1]
http_pos = lien.find("://")
if http_pos == -1:
url = lien
else:
url = lien[(http_pos+3):]
port_pos = url.find(':')
if port_pos == -1:
portserver = 80
else:
portserver = url[(port_pos+1):]
s_pos = url.find('/')
if s_pos == -1:
webserver = url
else:
webserver = url[:(s_pos)]
proxy_server(webserver, portserver, data, con)
def proxy_server(webserver, portserver, data, con):
print webserver
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((webserver, int(portserver)))
s.send(data)
while True:
red = s.recv(8192)
if len(red) > 0:
con.send(red)
start()
this is one of the error messages that i have !
Unhandled exception in thread started by <function conn_string at 0x0248CEF0>
Traceback (most recent call last):
File "C:\Users\none2\Desktop\Nouveau dossier\Target.py", line 52, in conn_stri
ng
proxy_server(webserver, portserver, data, con)
File "C:\Users\none2\Desktop\Nouveau dossier\Target.py", line 57, in proxy_ser
ver
s.connect((webserver, int(portserver)))
File "C:\Python27\lib\socket.py", line 228, in meth
return getattr(self._sock,name)(*args)
socket.gaierror: [Errno 11004] getaddrinfo failed

can you replace
start_new_thread(conn_string, (data, connection))
line with
start_new_thread(conn_string(data, connection))
tried running the same file, it seems above one is the only error

Unable to ssh to remote server using paramiko module

I am trying to ssh to a remote server using python paramiko module. I need to include the key file dynamically. My code is given below.
import getpass
import paramiko
server = raw_input("What is the server name? ")
username = raw_input("Enter the username: ")
passphrase = getpass.getpass(prompt="Enter your passphrase: ")
key = '/home/%s/.ssh/id_rsa' % username
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(server, username=username, password=passphrase, key_filename=key)
stdin, stdout, stderr = ssh.exec_command('df -h')
print stdout.readlines()
ssh.close()
I am able to work with the code if I provide the key path directly instead of using the variable.
The error I am getting is:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/lib/python2.7/dist-packages/paramiko/client.py", line 237, in connect
for (family, socktype, proto, canonname, sockaddr) in socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM):
socket.gaierror: [Errno -2] Name or service not known`enter code here`

seems like you have some dns error here, Pasting my script to get ssh status over here, that is dealing all the exceptions (at least I have noted so far)
#!/bin/python3
import threading, time, paramiko, socket, getpass
from queue import Queue
locke1 = threading.Lock()
q = Queue()
#Check the login
def check_hostname(host_name, pw_r):
with locke1:
print ("Checking hostname :"+str(host_name)+" with " + threading.current_thread().name)
file_output = open('output_file','a')
file_success = open('success_file','a')
file_failed = open('failed_file','a')
file_error = open('error_file','a')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(host_name, username='root', password=pw_r, timeout=5)
#print ("Success")
file_success.write(str(host_name+"\n"))
file_success.close()
file_output.write("success: "+str(host_name+"\n"))
file_output.close()
# printing output if required from remote machine
#stdin,stdout,stderr = ssh.exec_command("hostname&&uptime")
#for line in stdout.readlines():
# print (line.strip())
except paramiko.SSHException:
# print ("error")
file_failed.write(str(host_name+"\n"))
file_failed.close()
file_output.write("failed: "+str(host_name+"\n"))
file_output.close()
#quit()
except paramiko.ssh_exception.NoValidConnectionsError:
#print ("might be windows------------")
file_output.write("failed: " + str(host_name + "\n"))
file_output.close()
file_failed.write(str(host_name+"\n"))
file_failed.close()
#quit()
except socket.gaierror:
#print ("wrong hostname/dns************")
file_output.write("error: "+str(host_name+"\n"))
file_output.close()
file_error.write(str(host_name + "\n"))
file_error.close()
except socket.timeout:
#print ("No Ping %%%%%%%%%%%%")
file_output.write("error: "+str(host_name+"\n"))
file_output.close()
file_error.write(str(host_name + "\n"))
file_error.close()
ssh.close()
def performer1():
while True:
hostname_value = q.get()
check_hostname(hostname_value,pw_sent)
q.task_done()
if __name__ == '__main__':
print ("This script checks all the hostnames in the input_file with your standard password and write the outputs in below files: \n1.file_output\n2.file_success \n3.file_failed \n4.file_error \n")
f = open('output_file', 'w')
f.write("-------Output of all hosts-------\n")
f.close()
f = open('success_file', 'w')
f.write("-------Success hosts-------\n")
f.close()
f = open('failed_file', 'w')
f.write("-------Failed hosts-------\n")
f.close()
f = open('error_file', 'w')
f.write("-------Hosts with error-------\n")
f.close()
with open("input_file") as f:
hostname1 = f.read().splitlines()
#Read the standard password from the user
pw_sent=getpass.getpass("Enter the Password:")
start_time1 = time.time()
for i in hostname1:
q.put(i)
#print ("all the hostname : "+str(list(q.queue)))
for no_of_threads in range(10):
t = threading.Thread(target=performer1)
t.daemon=True
t.start()
q.join()
print ("Check output files for results")
print ("completed task in" + str(time.time()-start_time1) + "seconds")