I have to automatically upload folders to an FTP using a Python script. I am able to upload a single file, but not folders with subfolders and files in them. I did a lot of search, but failed. Could some one help me out here? Thanks in advance.
#! /usr/bin/python
import ftplib
s = ftplib.FTP('serverip','usrname','password')
file = '/home/rock/test.txt'
ftppath = '/IT'
filename = "rak"
s.cwd(ftppath)
f = open(file,'rb')
s.storbinary('STOR ' + filename, f)
f.close()
s.quit()
I recently came into this problem and figured out a recursive function to solve it.
import ftplib
import os
server = 'localhost'
username = 'generic_user'
password = 'password'
myFTP = ftplib.FTP(server, username, password)
myPath = r'c:\temp'
def uploadThis(path):
files = os.listdir(path)
os.chdir(path)
for f in files:
if os.path.isfile(path + r'\{}'.format(f)):
fh = open(f, 'rb')
myFTP.storbinary('STOR %s' % f, fh)
fh.close()
elif os.path.isdir(path + r'\{}'.format(f)):
myFTP.mkd(f)
myFTP.cwd(f)
uploadThis(path + r'\{}'.format(f))
myFTP.cwd('..')
os.chdir('..')
uploadThis(myPath) # now call the recursive function
You basically need to use os.walk() to grab those files and transfer them.
Here's a script I wrote for myself to do much of what your asking.
I wrote it a long time ago, so I'd probably do it differently if I wrote it again, but I get a lot of use out of it.
It imports psftplib, which is a wrapper I wrote for the putty sftp.
Feel free to remove these references, or grab the lib at:
http://code.google.com/p/psftplib/source/browse/trunk/psftplib.py
# -*- coding: utf8 -*-
'''This tool will ftp all the files in a given directory to a given location
if the file ftpallcfg.py exists in the directory it will be loaded and the values within it used,
with the current directory used as the source directory.
ftpallcfg.py file contains the following variables.
===========================
server = <server to ftp to>
username = <Username for access to given server>
remote_dir = <remote server directory>
encrypt= True/False
monitor = True/False
walk = True/False
===========================
'''
import ftplib
import os
import getpass
import sys
import time
import socket
import psftplib
__revision__ = 1.11
SLEEP_SECONDS = 1
class FtpAddOns():
PATH_CACHE = []
def __init__(self, ftp_h):
self.ftp_h = ftp_h
def ftp_exists(self, path):
'''path exists check function for ftp handler'''
exists = None
if path not in self.PATH_CACHE:
try:
self.ftp_h.cwd(path)
exists = True
self.PATH_CACHE.append(path)
except ftplib.error_perm, e:
if str(e.args).count('550'):
exists = False
else:
exists = True
return exists
def ftp_mkdirs(self, path, sep='/'):
'''mkdirs function for ftp handler'''
split_path = path.split(sep)
new_dir = ''
for server_dir in split_path:
if server_dir:
new_dir += sep + server_dir
if not self.ftp_exists(new_dir):
try:
print 'Attempting to create directory (%s) ...' % (new_dir),
self.ftp_h.mkd(new_dir)
print 'Done!'
except Exception, e:
print 'ERROR -- %s' % (str(e.args))
def _get_local_files(local_dir, walk=False):
'''Retrieve local files list
result_list == a list of dictionaries with path and mtime keys. ex: {'path':<filepath>,'mtime':<file last modified time>}
ignore_dirs == a list of directories to ignore, should not include the base_dir.
ignore_files == a list of files to ignore.
ignore_file_ext == a list of extentions to ignore.
'''
result_list = []
ignore_dirs = ['CVS', '.svn']
ignore_files = ['.project', '.pydevproject']
ignore_file_ext = ['.pyc']
base_dir = os.path.abspath(local_dir)
for current_dir, dirs, files in os.walk(base_dir):
for this_dir in ignore_dirs:
if this_dir in dirs:
dirs.remove(this_dir)
sub_dir = current_dir.replace(base_dir, '')
if not walk and sub_dir:
break
for this_file in files:
if this_file not in ignore_files and os.path.splitext(this_file)[-1].lower() not in ignore_file_ext:
filepath = os.path.join(current_dir, this_file)
file_monitor_dict = {
'path': filepath,
'mtime': os.path.getmtime(filepath)
}
result_list.append(file_monitor_dict)
return result_list
def monitor_and_ftp(server,
username,
password,
local_dir,
remote_dir,
encrypt=False,
walk=False):
'''Monitor local files and when an update is found connect and upload'''
print 'Monitoring changes in (%s).' % (os.path.abspath(local_dir))
print '(Use ctrl-c to exit)'
last_files_list = _get_local_files(local_dir)
while True:
try:
time.sleep(SLEEP_SECONDS)
latest_files_list = _get_local_files(local_dir)
files_to_update = []
for idx in xrange(len(latest_files_list)):
if idx < len(last_files_list):
# compare last modified times
if latest_files_list[idx]['mtime'] > last_files_list[idx]['mtime']:
files_to_update.append(latest_files_list[idx])
else:
# add the file to the list (new file)
files_to_update.append(latest_files_list[idx])
if files_to_update:
print
print 'Detected NEW or CHANGED file(s), attempting to send ...'
print
is_success = upload_all(server,
username,
password,
local_dir,
remote_dir,
files_to_update,
encrypt,
walk)
if not is_success:
break
else:
print '.',
last_files_list = latest_files_list[:] # copy the list to hold
except KeyboardInterrupt:
print
print 'Exiting.'
break
def upload_all(server,
username,
password,
base_local_dir,
base_remote_dir,
files_to_update=None,
encrypt=False,
walk=False):
'''Upload all files in a given directory to the given remote directory'''
continue_on = False
login_ok = False
server_connect_ok = False
base_local_dir = os.path.abspath(base_local_dir)
base_remote_dir = os.path.normpath(base_remote_dir)
if files_to_update:
local_files = files_to_update
else:
local_files = _get_local_files(base_local_dir, walk)
if local_files:
if not encrypt: # Use standard FTP
ftp_h = ftplib.FTP()
else: # Use sftp
ftp_h = psftplib.SFTP()
try:
ftp_h.connect(server)
server_connect_ok = True
except socket.gaierror, e:
print 'ERROR -- Could not connect to (%s): %s' % (server, str(e.args))
except IOError, e:
print 'ERROR -- File not found: %s' % (str(e.args))
except socket.error, e:
print 'ERROR -- Could not connect to (%s): %s' % (server, str(e.args))
ftp_path_tools = FtpAddOns(ftp_h)
if server_connect_ok:
try:
ftp_h.login(username,password)
print 'Logged into (%s) as (%s)' % (server, username)
login_ok = True
except ftplib.error_perm, e:
print 'ERROR -- Check Username/Password: %s' % (str(e.args))
except psftplib.ProcessTimeout, e:
print 'ERROR -- Check Username/Password (timeout): %s' % (str(e.args))
if login_ok:
for file_info in local_files:
filepath = file_info['path']
path, filename = os.path.split(filepath)
remote_sub_path = path.replace(base_local_dir, '')
remote_path = path.replace(base_local_dir, base_remote_dir)
remote_path = remote_path.replace('\\', '/') # Convert to unix style
if not ftp_path_tools.ftp_exists(remote_path):
ftp_path_tools.ftp_mkdirs(remote_path)
# Change to directory
try:
ftp_h.cwd(remote_path)
continue_on = True
except ftplib.error_perm, e:
print 'ERROR -- %s' % (str(e.args))
except psftplib.PsFtpInvalidCommand, e:
print 'ERROR -- %s' % (str(e.args))
if continue_on:
if os.path.exists(filepath):
f_h = open(filepath,'rb')
filename = os.path.split(f_h.name)[-1]
display_filename = os.path.join(remote_sub_path, filename)
display_filename = display_filename.replace('\\', '/')
print 'Sending (%s) ...' % (display_filename),
send_cmd = 'STOR %s' % (filename)
try:
ftp_h.storbinary(send_cmd, f_h)
f_h.close()
print 'Done!'
except Exception, e:
print 'ERROR!'
print str(e.args)
print
else:
print "WARNING -- File no longer exists, (%s)!" % (filepath)
ftp_h.quit()
print 'Closing Connection'
else:
print 'ERROR -- No files found in (%s)' % (base_local_dir)
return continue_on
if __name__ == '__main__':
import optparse
default_config_file = u'ftpallcfg.py'
# Create parser, and configure command line options to parse
parser = optparse.OptionParser()
parser.add_option("-l", "--local_dir",
dest="local_dir",
help="Local Directory (Defaults to CWD)",
default='.')
parser.add_option("-r", "--remote_dir",
dest="remote_dir",
help="[REQUIRED] Target Remote directory",
default=None)
parser.add_option("-u", "--username",
dest="username",
help="[REQUIRED] username",
default=None)
parser.add_option("-s","--server",
dest="server",
help="[REQUIRED] Server Address",
default=None)
parser.add_option("-e", "--encrypt",
action="store_true",
dest="encrypt",
help="Use sftp",
default=False)
parser.add_option("-m",
action="store_true",
dest="monitor",
help="Keep process open and monitor changes",
default=False)
parser.add_option("-w",
action="store_true",
dest="walkdir",
help="Walk sub directories of the given directory to find files to send.",
default=False)
(options,args) = parser.parse_args()
if (options.username and options.server and options.remote_dir) or \
os.path.exists(default_config_file):
local_dir = options.local_dir
if os.path.exists(default_config_file):
sys.path.append('.')
import ftpallcfg
try:
server = ftpallcfg.server
username = ftpallcfg.username
remote_dir = ftpallcfg.remote_dir
encrypt = ftpallcfg.encrypt
monitor = ftpallcfg.monitor
walk = ftpallcfg.walk
except AttributeError, e:
print "ERROR --", str(e.args)
print
print 'Value(s) missing in %s file! The following values MUST be included:' % (default_config_file)
print '================================'
print 'server = <server to ftp to>'
print 'username = <Username for access to given server>'
print 'remote_dir = <remote server directory>'
print 'encrypt= True/False'
print 'monitor = True/False'
print 'walk == True/False'
print '================================'
sys.exit()
else:
server = options.server
username = options.username
remote_dir = options.remote_dir
encrypt = options.encrypt
monitor = options.monitor
walk = options.walkdir
# get the user password
prompt = 'Password (%s#%s): ' % (username, server)
if os.isatty(sys.stdin.fileno()):
p = getpass.getpass(prompt)
else:
#p = sys.stdin.readline().rstrip()
p = raw_input(prompt).rstrip()
if options.encrypt:
print '>> Using sftp for secure transfers <<'
print
if monitor:
try:
monitor_and_ftp(server,username,p,local_dir, remote_dir, encrypt, walk)
except KeyboardInterrupt:
print 'Exiting...'
else:
try:
upload_all(server, username, p, local_dir, remote_dir, [], encrypt, walk)
except KeyboardInterrupt:
print 'Exiting...'
else:
print 'ERROR -- Required option not given!'
print __revision__
print __doc__
print
parser.print_help()
EDIT 20/12/2017:
I have written a project in GitHub for this purpose. Click for details!
There are good answers above but i also want to add a good one using ftputil package. If you need to upload files from local directory to ftp directory, you can use this recursive function:
def upload_dir(localDir, ftpDir):
list = os.listdir(localDir)
for fname in list:
if os.path.isdir(localDir + fname):
if(ftp_host.path.exists(ftpDir + fname) != True):
ftp_host.mkdir(ftpDir + fname)
print(ftpDir + fname + " is created.")
upload_dir(localDir + fname + "/", ftpDir + fname + "/")
else:
if(ftp_host.upload_if_newer(localDir + fname, ftpDir + fname)):
print(ftpDir + fname + " is uploaded.")
else:
print(localDir + fname + " has already been uploaded.")
If you decide to use this function, you have to connect ftp using ftputil package. For this, you can use this snippet:
with ftputil.FTPHost("ftp_host", "ftp_username", "ftp_password") as ftp_host:
So, we're almost done. The last thing is usage of the function for beginners like me:
local_dir = "D:/Projects/.../"
ftp_dir = "/.../../"
upload_dir(local_dir, ftp_dir)
The most important thing is "/" character at the end of paths. You need to put it at the end. Finally, i want to share entire code:
with ftputil.FTPHost("ftp_host", "ftp_username", "ftp_password") as ftp_host:
def upload_dir(localDir, ftpDir):
list = os.listdir(localDir)
for fname in list:
if os.path.isdir(localDir + fname):
if(ftp_host.path.exists(ftpDir + fname) != True):
ftp_host.mkdir(ftpDir + fname)
print(ftpDir + fname + " is created.")
upload_dir(localDir + fname + "/", ftpDir + fname + "/")
else:
if(ftp_host.upload_if_newer(localDir + fname, ftpDir + fname)):
print(ftpDir + fname + " is uploaded.")
else:
print(localDir + fname + " has already been uploaded.")
local_dir = "D:/Projects/.../"
ftp_dir = "/.../../"
upload_dir(local_dir, ftp_dir)
Maybe you try ftpsync.py. If this one doesn't helps, try google search on python ftpsync and you get a lot of answers.
using ftputil:
import os
import ftputil
import ftputil.session
def upload_dir(root):
root = unicode(root, 'utf-8')
for dir_name, _, dir_files in os.walk(root):
local = os.path.join(os.curdir, dir_name)
remote = ftp_host.path.join(ftp_host.curdir, dir_name)
if not ftp_host.path.exists(remote):
print 'mkdir:', local, '->', remote
ftp_host.mkdir(remote)
for f in dir_files:
local_f = os.path.join(local, f)
remote_f = ftp_host.path.join(remote, f)
print 'upload:', local_f, '->', remote_f
ftp_host.upload(local_f, remote_f)
sf = ftputil.session.session_factory(use_passive_mode=True)
with ftputil.FTPHost('HOST', 'USER', 'PASS', session_factory=sf) as ftp_host:
upload_dir('DIR')
It is easy to use lftp to upload folders to an FTP. I use this in my Python script to move folders to FTP
Python script:
#! /usr/bin/python
import subprocess
subprocess.call(["bash", ftp_script, username, password, ftp, folder_to_move, src,folder_name_in_destination])
ftp_script:
lftp -u $1,$2 $3 <<EOF
mkdir $4
lcd $5
cd $6
mirror --reverse
EOF
Yet another answer using ftputil. And pathlib for figuring out the local path.
I've compiled and improved some of the above answers.
So thank you for the inspiration!
import os
import ftplib
import ftputil
from pathlib import Path
def ftp_upload(ftp_host, src_path, dst_path=None):
if dst_path is None:
ftp_host.path.join(os.curdir)
# Make sure that local path is a directory
if src_path.is_dir():
# Make sure that the root path exists
if ftp_host.path.exists(dst_path) is False:
# Inform user
print(f'mkdir: {src_path} -> {dst_path}')
# Create folder
ftp_host.mkdir(dst_path)
# Parse content of the root folder
for src_child in src_path.iterdir():
if src_child.is_dir():
# Use recursion for sub folders/files
dst_child = ftp_host.path.join(dst_path, src_child.stem)
ftp_upload(ftp_host, src_child, dst_child)
else:
# Copy sub files
dst_child = ftp_host.path.join(dst_path, src_child.name)
# Inform user
print(f'upload: {src_child} -> {dst_child}')
# Perform copy (upload)
ftp_host.upload(src_child, dst_child)
# ftp_host.upload_if_newer(src_child, dst_child)
else:
# Inform user
print(f"src_path: '{src_path}' must be an existing directory!")
if __name__ == '__main__':
# FTP settings
user = "ftp_user"
password = "ftp_password"
host = "localhost"
port = 2121
# Path settings
src_path = Path(__file__).parent.resolve() / "upload_dir"
dst_dir = "upload_dir"
# Inform user
print(f"Establishing FTP session as '{user}'")
# Establish FTP session
ftplib.FTP.port = port
with ftputil.FTPHost(host, user, password) as ftp_host:
# Perform recursive FTP upload
dst_path = ftp_host.path.join(os.curdir, dst_dir)
ftp_upload(ftp_host, src_path, dst_path)
Related
I'm trying to get a database backup from odoo erp using ftputil
this is the code
# -*- coding: utf-8 -*-
from odoo import models, fields, api, tools, _
from odoo.exceptions import Warning
import odoo
from odoo.http import content_disposition
import logging
_logger = logging.getLogger(__name__)
import os
import datetime
try:
from xmlrpc import client as xmlrpclib
except ImportError:
import xmlrpclib
import time
import base64
import socket
try:
import ftputil
except ImportError:
raise ImportError(
'This module needs ftputil to automatically write backups to the FTP through ftp. Please install ftputil on your system. (sudo pip3 install ftputil)')
def execute(connector, method, *args):
res = False
try:
res = getattr(connector, method)(*args)
except socket.error as error:
_logger.critical('Error while executing the method "execute". Error: ' + str(error))
raise error
return res
class db_backup(models.Model):
_name = 'db.backup'
#api.multi
def get_db_list(self, host, port, context={}):
uri = 'http://' + host + ':' + port
conn = xmlrpclib.ServerProxy(uri + '/xmlrpc/db')
db_list = execute(conn, 'list')
return db_list
#api.multi
def _get_db_name(self):
dbName = self._cr.dbname
return dbName
# Columns for local server configuration
host = fields.Char('Host', required=True, default='localhost')
port = fields.Char('Port', required=True, default=8069)
name = fields.Char('Database', required=True, help='Database you want to schedule backups for',
default=_get_db_name)
folder = fields.Char('Backup Directory', help='Absolute path for storing the backups', required='True',
default='/odoo/backups')
backup_type = fields.Selection([('zip', 'Zip'), ('dump', 'Dump')], 'Backup Type', required=True, default='zip')
autoremove = fields.Boolean('Auto. Remove Backups',
help='If you check this option you can choose to automaticly remove the backup after xx days')
days_to_keep = fields.Integer('Remove after x days',
help="Choose after how many days the backup should be deleted. For example:\nIf you fill in 5 the backups will be removed after 5 days.",
required=True)
# Columns for external server (SFTP)
sftp_write = fields.Boolean('Write to external server with sftp',
help="If you check this option you can specify the details needed to write to a remote server with SFTP.")
sftp_path = fields.Char('Path external server',
help='The location to the folder where the dumps should be written to. For example /odoo/backups/.\nFiles will then be written to /odoo/backups/ on your remote server.')
sftp_host = fields.Char('IP Address SFTP Server',
help='The IP address from your remote server. For example 192.168.0.1')
sftp_port = fields.Integer('SFTP Port', help='The port on the FTP server that accepts SSH/SFTP calls.', default=22)
sftp_user = fields.Char('Username SFTP Server',
help='The username where the SFTP connection should be made with. This is the user on the external server.')
sftp_password = fields.Char('Password User SFTP Server',
help='The password from the user where the SFTP connection should be made with. This is the password from the user on the external server.')
days_to_keep_sftp = fields.Integer('Remove SFTP after x days',
help='Choose after how many days the backup should be deleted from the FTP server. For example:\nIf you fill in 5 the backups will be removed after 5 days from the FTP server.',
default=30)
send_mail_sftp_fail = fields.Boolean('Auto. E-mail on backup fail',
help='If you check this option you can choose to automaticly get e-mailed when the backup to the external server failed.')
email_to_notify = fields.Char('E-mail to notify',
help='Fill in the e-mail where you want to be notified that the backup failed on the FTP.')
#api.multi
def _check_db_exist(self):
self.ensure_one()
db_list = self.get_db_list(self.host, self.port)
if self.name in db_list:
return True
return False
_constraints = [(_check_db_exist, _('Error ! No such database exists!'), [])]
#api.multi
def test_sftp_connection(self, context=None):
self.ensure_one()
# Check if there is a success or fail and write messages
messageTitle = ""
messageContent = ""
error = ""
has_failed = False
for rec in self:
db_list = self.get_db_list(rec.host, rec.port)
pathToWriteTo = rec.sftp_path
ipHost = rec.sftp_host
portHost = rec.sftp_port
usernameLogin = rec.sftp_user
passwordLogin = rec.sftp_password
# Connect with external server over SFTP, so we know sure that everything works.
try:
with ftputil.FTPHost(ipHost, usernameLogin, passwordLogin) as s:
messageTitle = _("Connection Test Succeeded!\nEverything seems properly set up for FTP back-ups!")
except Exception as e:
_logger.critical('There was a problem connecting to the remote ftp: ' + str(e))
error += str(e)
has_failed = True
messageTitle = _("Connection Test Failed!")
if len(rec.sftp_host) < 8:
messageContent += "\nYour IP address seems to be too short.\n"
messageContent += _("Here is what we got instead:\n")
finally:
if s:
s.close()
if has_failed:
raise Warning(messageTitle + '\n\n' + messageContent + "%s" % str(error))
else:
raise Warning(messageTitle + '\n\n' + messageContent)
#api.model
def schedule_backup(self):
conf_ids = self.search([])
for rec in conf_ids:
db_list = self.get_db_list(rec.host, rec.port)
if rec.name in db_list:
try:
if not os.path.isdir(rec.folder):
os.makedirs(rec.folder)
except:
raise
# Create name for dumpfile.
bkp_file = '%s_%s.%s' % (time.strftime('%Y_%m_%d_%H_%M_%S'), rec.name, rec.backup_type)
file_path = os.path.join(rec.folder, bkp_file)
uri = 'http://' + rec.host + ':' + rec.port
conn = xmlrpclib.ServerProxy(uri + '/xmlrpc/db')
bkp = ''
try:
# try to backup database and write it away
fp = open(file_path, 'wb')
odoo.service.db.dump_db(rec.name, fp, rec.backup_type)
fp.close()
except Exception as error:
_logger.debug(
"Couldn't backup database %s. Bad database administrator password for server running at http://%s:%s" % (
rec.name, rec.host, rec.port))
_logger.debug("Exact error from the exception: " + str(error))
continue
else:
_logger.debug("database %s doesn't exist on http://%s:%s" % (rec.name, rec.host, rec.port))
# Check if user wants to write to SFTP or not.
if rec.sftp_write is True:
try:
# Store all values in variables
dir = rec.folder
pathToWriteTo = rec.sftp_path
ipHost = rec.sftp_host
portHost = rec.sftp_port
usernameLogin = rec.sftp_user
passwordLogin = rec.sftp_password
_logger.debug('sftp remote path: %s' % pathToWriteTo)
try:
with ftputil.FTPHost(ipHost, usernameLogin, passwordLogin) as sftp:
pass
except Exception as error:
_logger.critical('Error connecting to remote server! Error: ' + str(error))
try:
sftp.chdir(pathToWriteTo)
except IOError:
# Create directory and subdirs if they do not exist.
currentDir = ''
for dirElement in pathToWriteTo.split('/'):
currentDir += dirElement + '/'
try:
sftp.chdir(currentDir)
except:
_logger.info('(Part of the) path didn\'t exist. Creating it now at ' + currentDir)
# Make directory and then navigate into it
sftp.mkdir(currentDir, 777)
sftp.chdir(currentDir)
pass
sftp.chdir(pathToWriteTo)
# Loop over all files in the directory.
for f in os.listdir(dir):
if rec.name in f:
fullpath = os.path.join(dir, f)
if os.path.isfile(fullpath):
try:
sftp.StatResult(os.path.join(pathToWriteTo, f))
_logger.debug(
'File %s already exists on the remote FTP Server ------ skipped' % fullpath)
# This means the file does not exist (remote) yet!
except IOError:
try:
# sftp.put(fullpath, pathToWriteTo)
sftp.upload(fullpath, os.path.join(pathToWriteTo, f))
_logger.info('Copying File % s------ success' % fullpath)
except Exception as err:
_logger.critical(
'We couldn\'t write the file to the remote server. Error: ' + str(err))
# Navigate in to the correct folder.
sftp.chdir(pathToWriteTo)
# Loop over all files in the directory from the back-ups.
# We will check the creation date of every back-up.
for file in sftp.listdir(pathToWriteTo):
if rec.name in file:
# Get the full path
fullpath = os.path.join(pathToWriteTo, file)
# Get the timestamp from the file on the external server
timestamp = sftp.StatResult(fullpath).st_atime
createtime = datetime.datetime.fromtimestamp(timestamp)
now = datetime.datetime.now()
delta = now - createtime
# If the file is older than the days_to_keep_sftp (the days to keep that the user filled in on the Odoo form it will be removed.
if delta.days >= rec.days_to_keep_sftp:
# Only delete files, no directories!
if sftp.isfile(fullpath) and (".dump" in file or '.zip' in file):
_logger.info("Delete too old file from SFTP servers: " + file)
sftp.unlink(file)
# Close the SFTP session.
sftp.close()
except Exception as e:
_logger.debug('Exception! We couldn\'t back up to the FTP server..')
# At this point the SFTP backup failed. We will now check if the user wants
# an e-mail notification about this.
if rec.send_mail_sftp_fail:
try:
ir_mail_server = self.env['ir.mail_server']
message = "Dear,\n\nThe backup for the server " + rec.host + " (IP: " + rec.sftp_host + ") failed.Please check the following details:\n\nIP address SFTP server: " + rec.sftp_host + "\nUsername: " + rec.sftp_user + "\nPassword: " + rec.sftp_password + "\n\nError details: " + tools.ustr(
e) + "\n\nWith kind regards"
msg = ir_mail_server.build_email("auto_backup#" + rec.name + ".com", [rec.email_to_notify],
"Backup from " + rec.host + "(" + rec.sftp_host + ") failed",
message)
ir_mail_server.send_email(self._cr, self._uid, msg)
except Exception:
pass
"""
Remove all old files (on local server) in case this is configured..
"""
if rec.autoremove:
dir = rec.folder
# Loop over all files in the directory.
for f in os.listdir(dir):
fullpath = os.path.join(dir, f)
# Only delete the ones which are from the current database
# (Makes it possible to save different databases in the same folder)
if rec.name in fullpath:
timestamp = os.stat(fullpath).st_ctime
createtime = datetime.datetime.fromtimestamp(timestamp)
now = datetime.datetime.now()
delta = now - createtime
if delta.days >= rec.days_to_keep:
# Only delete files (which are .dump and .zip), no directories.
if os.path.isfile(fullpath) and (".dump" in f or '.zip' in f):
_logger.info("Delete local out-of-date file: " + fullpath)
os.remove(fullpath)
I cant get pass this logger "_logger.critical(
'We couldn't write the file to the remote server. Error: ' + str(err))"
I'm trying to take an image via webcam then send the image through gmail. That function suppose to happen if my fingerprint is known at the system. If it does it needs to execute the send email function. I added after the send email function to send 'HIGH' to a gpio. I don't think it suppose to interfere.
When there is a known template it is followed by an "else:" statement. then I put my "def find_newest_file(dir):" function. What am I doing wrong?
might note I'm terrible at programming. Thank you!
Pyfingerprint
import os
import glob
import time
import smtplib
import imghdr
from email.message import EmailMessage
import RPi.GPIO as GPIO
from time import sleep
GPIO.setmode(GPIO.BCM)
GPIO.setup(26, GPIO.OUT)
import hashlib
from pyfingerprint.pyfingerprint import PyFingerprint
GPIO.output(26, 0)
## Search for a finger
##
## Tries to initialize the sensor
try:
f = PyFingerprint('/dev/ttyUSB0', 57600, 0xFFFFFFFF, 0x00000000)
if ( f.verifyPassword() == False ):
raise ValueError('The given fingerprint sensor password is wrong!')
except Exception as e:
print('The fingerprint sensor could not be initialized!')
print('Exception message: ' + str(e))
exit(1)
## Gets some sensor information
print('Currently used templates: ' + str(f.getTemplateCount()) +'/'+ str(f.getStorageCapacity()))
## Tries to search the finger and calculate hash
try:
print('Waiting for finger...')
## Wait that finger is read
while ( f.readImage() == False ):
pass
## Converts read image to characteristics and stores it in charbuffer 1
f.convertImage(0x01)
## Searchs template
result = f.searchTemplate()
positionNumber = result[0]
accuracyScore = result[1]
if ( positionNumber == -1 ):
print('No match found!')
exit(0)
else:
print('Found template at position #' + str(positionNumber))
print('The accuracy score is: ' + str(accuracyScore))
def find_newest_file(dir):
os.system('fswebcam -r 1280x720 -S 3 --jpeg 90 --save /home/pi/Pictures/%H%M%S.jpg')
types = ['*.png', '*.jpg']
files = []
if not os.path.isdir(dir):
print(f'ERROR: {dir} does not exist. or it is not a valid folder')
exit()
for ext in types:
scan_path = os.path.abspath(os.path.join(dir, ext))
files.extend(glob.glob(scan_path))
if len(files) == 0:
print(f'ERROR: file not found while scanning folder: {dir} for: {types}')
exit()
newest = None
n_time = 0
for file in files:
# print(file)
c_time = os.path.getctime(file)
if c_time > n_time:
n_time = c_time
newest = file
if newest is None:
print(f'-----------\nUnexpected error: None was return while the list was not empty:\n{files}')
exit()
if os.path.exists(newest):
return newest # return as a list since this is what Yehonata expect
else:
print(f'ERROR: File {newest} not found')
exit()
# Yehontan Code
Sender_Email = "Sender#gmail.com"
Reciever_Email = "reciver#gmailcom"
Password = input('Enter your email account password: ')
newMessage = EmailMessage()
newMessage['Subject'] = "Check out the new logo"
newMessage['From'] = Sender_Email
newMessage['To'] = Reciever_Email
newMessage.set_content('Let me know what you think. Image attached!')
# Tomerl: Replace static name with seach function
files = [ find_newest_file('/home/pi/Pictures/') ]
for file in files:
with open(file, 'rb') as f:
image_data = f.read()
image_type = imghdr.what(f.name)
image_name = f.name
newMessage.add_attachment(image_data, maintype='image', subtype=image_type, filename=image_name)
with smtplib.SMTP_SSL('smtp.gmail.com', 465) as smtp:
smtp.login(Sender_Email, Password)
smtp.send_message(newMessage)
try:
for x in range(1):
GPIO.output(26, 1)
sleep(2.5)
GPIO.output(26, 0)
sleep(2.5)
except KeyboardInterrupt:
GPIO.cleanup()
GPIO.output(26, 0)
## Loads the found template to charbuffer 1
f.loadTemplate(positionNumber, 0x01)
## Downloads the characteristics of template loaded in charbuffer 1
characterics = str(f.downloadCharacteristics(0x01)).encode('utf-8')
## Hashes characteristics of template
print('SHA-2 hash of template: ' + hashlib.sha256(characterics).hexdigest())
except Exception as e:
print('Operation failed!')
print('Exception message: ' + str(e))
exit(1)
You need to define the function at the top of the program. You only need to call it after the else statement. Basically, put
def find_newest_file(dir):
os.system('fswebcam -r 1280x720 -S 3 --jpeg 90 --save /home/pi/Pictures/%H%M%S.jpg')
types = ['*.png', '*.jpg']
files = []
if not os.path.isdir(dir):
print(f'ERROR: {dir} does not exist. or it is not a valid folder')
exit()
at the top and when you want to use it do
find_newest_file(foo)
Good luck on your journey!
I am working on a simple zip file password cracker for a school project, and I need it to display the password once it cracks it from the dictionary word list. Whenever I run it, it only extracts the file, and doesn't print anything. How can I fix this to also show the password? Here is my code.
import optparse
import zipfile
from threading import Thread
def extract_zip(zFile, password):
try:
password = bytes(password.encode('utf-8'))
zFile.extractall(pwd=password)
print ("[+] Password Found: " + password + '\n')
except:
pass
def Main():
parser = optparse.OptionParser("useage &prog "+\
"-f <zipfile> -d <dictionary>")
parser.add_option('-f', dest='zname', type='string',\
help='specify zip file')
parser.add_option('-d', dest='dname', type='string',\
help='specify dictionary file')
(options, arg) = parser.parse_args()
if (options.zname == None) | (options.dname == None):
print (parser.usage)
exit(0)
else:
zname = options.zname
dname = options.dname
zFile = zipfile.ZipFile(zname)
passFile = open(dname)
for line in passFile.readlines():
password = line.strip('\n')
t = Thread(target=extract_zip, args=(zFile, password))
t.start()
if __name__ == '__main__':
Main()
The problem is that you're trying to print the encoded password instead of the original password. You can't concatenate bytes to a string. So print the original password, not the result of bytes().
And instead of extracting all the files from the archive, use testzip() to test whether you can decrypt them. But to do this, each thread needs its own ZipFile object. Otherwise they'll set the password used by another thread.
def extract_zip(filename, password):
with ZipFile(filename) as zFile:
try:
password_encoded = bytes(password.encode('utf-8'))
zFile.setpassword(password_encoded)
zFile.testzip()
print ("[+] Password Found: " + password + '\n')
except:
pass
Then change the caller to pass the filename to the thread, not zFile.
import zipfile
from tqdm import tqdm
def chunck(fd,size=65536):
while 1:
x=fd.read(size)
if not x:
break
yield x
def file_len(path):
with open(path,'r',encoding='utf-8',errors='ignore') as fd:
return sum(x.count('\n') for x in chunck(fd))
def linear_zip_crack(zip_path,pwd_path):
ln=file_len(pwd_path)
zp=zipfile.ZipFile(zip_path)
with open(pwd_path,'rb') as fd:
for x in tqdm(fd,total=ln,unit='x'):
try:
zp.extractall(pwd=x.strip())
except:
continue
else:
print(f'pwd={x.decode().strip()}')
exit(0)
print('Not found')
linear_zip_crack('spn.zip','pwds.txt')
I am writing a python script to get logged in using ftp and download a file.But whenever I run this script,it says I have provided wrong user name or passwd.I am inputting right password still i am unable to run this script.My code is:
import os,getpass
from urllib.request import urlopen
filename='68544.jpg'
password=getpass.getpass('??')
At this line below the script is failed to run and whenever i run this address in browser it runs fine.
remoteaddr='ftp://Kamal:%s#localhost/%s;type=i'%(password,filename)
remotefile=urlopen(remoteaddr)
localfile=open(filename,'wb')
localfile.write(remotefile.read())
localfile.close()
remotefile.close()
def ftp_connect(path):
link = FTP(host = 'example.com', timeout = 5) #Keep low timeout
link.login(passwd = 'ftppass', user = 'ftpuser')
debug("%s - Connected to FTP" % strftime("%d-%m-%Y %H.%M"))
link.cwd(path)
return link
downloaded = open('/local/path/to/file.tgz', 'wb')
def debug(txt):
print txt
link = ftp_connect(path)
file_size = link.size(filename)
max_attempts = 5 #I dont want death loops.
while file_size != downloaded.tell():
try:
debug("%s while > try, run retrbinary\n" % strftime("%d-%m-%Y %H.%M"))
if downloaded.tell() != 0:
link.retrbinary('RETR ' + filename, downloaded.write, downloaded.tell())
else:
link.retrbinary('RETR ' + filename, downloaded.write)
except Exception as myerror:
if max_attempts != 0:
debug("%s while > except, something going wrong: %s\n \tfile lenght is: %i > %i\n"(strftime("%d-%m-%Y %H.%M"), myerror, file_size, downloaded.tell()))
link = ftp_connect(path)
max_attempts -= 1
else:
break
debug("Done with file, attempt to download m5dsum")
[...]
Use Paramiko library
import paramiko
paramiko.util.log_to_file('/tmp/paramiko.log')
# Open a transport
host = "example.com"
port = 22
transport = paramiko.Transport((host, port))
# Auth
password = "foo"
username = "bar"
transport.connect(username = username, password = password)
# Go!
sftp = paramiko.SFTPClient.from_transport(transport)
# Download
### It is relative path from the folder path on which this sftp user has default rights.
filepath = 'folder/file1.txt'
localpath = '/opt/backup/file.txt'
sftp.get(filepath, localpath)
# Close
sftp.close()
transport.close()
I have been trying to do some password cracking using L0phtcrack. I have a LDIF file that cannot be imported in to l0phtcrack as it is. lophtcrack does not also import from a csv file. The only option that I have using lophtcrack is to import from a Unix shadow file. Has anyone tried out converting a ldif file to a shadow file format in python ? If yes, would you mind sharing the script here.
Thanks
UPDATE - I made a few edits to code that i found online and got it to work. I now have a shadow file from my LDIF extract.
#!/usr/bin/env python
"""
Read in a ldap/ldif file and split out a unix-formated password file to run in
john the ripper for password auditing.
Typical usage (on the ldap server):
Dump the ldap database
slapcat > ldif.out
Convert it to passwd format
ldap-passwd-dump.py ldif.out passwd.out
Run john to crack passwords
john passwd.out
Aaron Peterson <aaron#midnightresearch.com>
"""
import sys, base64, re, os
class Results:
"""
Keep all user results here
"""
def __init__(self):
self.users = []
def addUser(self, user):
# Check for duplicates
dupe = 0
for u in self.users:
if u.uid == user.uid:
dupe = 1
break
if not dupe:
print " [*] Adding new user [%s, %s] to results" % (user.cn, user.uid)
self.users.append(user)
else:
print " [*] Not adding duplicate user [%s]" % user.cn
class User(list):
def __init__(self, hash=None, base64=None, password=None, cn=None, uid=None):
self.hash = hash
self.uid = uid
self.base64 = base64
self.password = password
self.cn = cn
list.__init__(self)
class LDIFCrack:
def main(self):
# Open file
f = open(self.ldif, "r")
# Load in the first user
user = User()
isInGroup=0
# Load lines into a "user"
for line in f:
if re.compile(r"^\s*$").search(line):
# Only append the old user if it's in the right group, and has a password set
if isInGroup and user.hash:
self.results.addUser(user)
# Reset user and counter
user = User()
isInGroup=0
# Make sure we test the right groups
if re.compile(self.groupMatch).search(line):
isInGroup=1
# Pull out the password
match = re.compile(r"userpassword: (.*)$").search(line)
if match:
user.hash = match.group(1)
# uid
match = re.compile(r"uid: (.*)$").search(line)
if match:
user.uid= match.group(1)
# Grab the common name
matchCn = re.compile(r"cn: (.*)$").search(line)
if matchCn:
user.cn = matchCn.group(1)
def printPasswd(self, file):
f = open(file, "w")
for user in self.results.users:
line = "%s:%s:::%s" % (user.uid, user.hash, user.cn)
f.write(line + "\n")
print " [*] %s" % line
f.close()
print " [*] Wrote [%s] password lines to [%s] " % (len(self.results.users), file)
def __init__(self, ldif, groupMatch):
self.ldif = ldif
self.results = Results()
self.groupMatch = groupMatch
self.main()
if __name__ == "__main__":
if len(sys.argv) < 3:
print "\nusage: %s <ldif file> <output password file> [<user matchString>]" % sys.argv[0]
print " example: %s ldif.out passwd.txt \"^ou: MyGroup\"" % sys.argv[0]
print " (matchString default is \"objectClass: posixAccount\")\n"
sys.exit(1)
ldif = sys.argv[1]
passwdFile = sys.argv[2]
if not os.path.exists(ldif):
print " [!] LDIF Input file [%s] does not exist..." % ldif
sys.exit(1)
if os.path.exists(passwdFile):
print " [!] Won't overwrite existing passwd file [%s]" % passwdFile
sys.exit(1)
# Will match the user against this group before cracking it if it's set
if len(sys.argv) == 4:
groupMatch = sys.argv[3]
else:
groupMatch = "objectClass: posixAccount"
ldifcrack = LDIFCrack(ldif, groupMatch)
ldifcrack.printPasswd(passwdFile)
print " [*] Done"