I am trying to troubleshoot a script for Mimecast's API. The script runs fine for the most part, but a few times, I have noticed that it stops pulling logs and generally appears to be a hung process. After restarting the script and manually pushing logs to the syslog server, it starts working again without issue. I am not able to reproduce this issue at will.
The script is supposed to do the following:
Authenticate against Mimecast's API
Sign responses
download, extract and save log files to log dir
utilize a tokenized header to determine which file was downloaded in the last request. Should save the token ID within a file in the checkpoint directory
Push files to remote syslog server
Output any errors and info to console
Below is the sample code from Mimecast.
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging.handlers
import json
import os
import requests
import base64
import uuid
import datetime
import hashlib
import shutil
import hmac
import time
from zipfile import ZipFile
import io
# Set up variables
APP_ID = "YOUR DEVELOPER APPLICATION ID"
APP_KEY = "YOUR DEVELOPER APPLICATION KEY"
URI = "/api/audit/get-siem-logs"
EMAIL_ADDRESS = 'EMAIL ADDRESS OF YOUR ADMINISTRATOR'
ACCESS_KEY = 'ACCESS KEY FOR YOUR ADMINISTRATOR'
SECRET_KEY = 'SECRET KEY FOR YOUR ADMINISTRATOR'
LOG_FILE_PATH = "FULLY QUALIFIED PATH TO FOLDER TO WRITE LOGS"
CHK_POINT_DIR = 'FULLY QUALIFIED PATH TO FOLDER TO WRITE PAGE TOKEN'
# Set True to output to syslog, false to only save to file
syslog_output = False
# Enter the IP address or hostname of your syslog server
syslog_server = 'localhost'
# Change this to override default port
syslog_port = 514
# delete files after fetching
delete_files = True
# Set threshold in number of files in log file directory
log_file_threshold = 10000
# Set up logging (in this case to terminal)
log = logging.getLogger(__name__)
log.root.setLevel(logging.DEBUG)
log_formatter = logging.Formatter('%(levelname)s %(message)s')
log_handler = logging.StreamHandler()
log_handler.setFormatter(log_formatter)
log.addHandler(log_handler)
# Set up syslog output
syslog_handler = logging.handlers.SysLogHandler(address=(syslog_server, syslog_port))
syslog_formatter = logging.Formatter('%(message)s')
syslog_handler.setFormatter(syslog_formatter)
syslogger = logging.getLogger(__name__)
syslogger = logging.getLogger('SysLogger')
syslogger.addHandler(syslog_handler)
# Supporting methods
def get_hdr_date():
return datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC")
def read_file(file_name):
try:
with open(file_name, 'r') as f:
data = f.read()
return data
except Exception as e:
log.error('Error reading file ' + file_name + '. Cannot continue. Exception: ' + str(e))
quit()
def write_file(file_name, data_to_write):
if '.zip' in file_name:
try:
byte_content = io.BytesIO(data_to_write)
zip_file = ZipFile(byte_content)
zip_file.extractall(LOG_FILE_PATH)
except Exception as e:
log.error('Error writing file ' + file_name + '. Cannot continue. Exception: ' + str(e))
quit()
else:
try:
with open(file_name, 'w') as f:
f.write(data_to_write)
except Exception as e:
log.error('Error writing file ' + file_name + '. Cannot continue. Exception: ' + str(e))
quit()
def get_base_url(email_address):
# Create post body for request
post_body = dict()
post_body['data'] = [{}]
post_body['data'][0]['emailAddress'] = email_address
# Create variables required for request headers
request_id = str(uuid.uuid4())
request_date = get_hdr_date()
headers = {'x-mc-app-id': APP_ID, 'x-mc-req-id': request_id, 'x-mc-date': request_date}
# Send request to API
log.debug('Sending request to https://api.mimecast.com/api/discover-authentication with request Id: ' +
request_id)
try:
r = requests.post(url='https://api.mimecast.com/api/login/discover-authentication',
data=json.dumps(post_body), headers=headers)
# Handle Rate Limiting
if r.status_code == 429:
log.warning('Rate limit hit. sleeping for ' + str(r.headers['X-RateLimit-Reset'] * 1000))
time.sleep(r.headers['X-RateLimit-Reset'] * 1000)
except Exception as e:
log.error('Unexpected error getting base url. Cannot continue.' + str(e))
quit()
# Handle error from API
if r.status_code != 200:
log.error('Request returned with status code: ' + str(r.status_code) + ', response body: ' +
r.text + '. Cannot continue.')
quit()
# Load response body as JSON
resp_data = json.loads(r.text)
# Look for api key in region region object to get base url
if 'region' in resp_data["data"][0]:
base_url = resp_data["data"][0]["region"]["api"].split('//')
base_url = base_url[1]
else:
# Handle no region found, likely the email address was entered incorrectly
log.error(
'No region information returned from API, please check the email address.'
'Cannot continue')
quit()
return base_url
def post_request(base_url, uri, post_body, access_key, secret_key):
# Create variables required for request headers
request_id = str(uuid.uuid4())
request_date = get_hdr_date()
unsigned_auth_header = '{date}:{req_id}:{uri}:{app_key}'.format(
date=request_date,
req_id=request_id,
uri=uri,
app_key=APP_KEY
)
hmac_sha1 = hmac.new(
base64.b64decode(secret_key),
unsigned_auth_header.encode(),
digestmod=hashlib.sha1).digest()
sig = base64.encodebytes(hmac_sha1).rstrip()
headers = {
'Authorization': 'MC ' + access_key + ':' + sig.decode(),
'x-mc-app-id': APP_ID,
'x-mc-date': request_date,
'x-mc-req-id': request_id,
'Content-Type': 'application/json'
}
try:
# Send request to API
log.debug('Sending request to https://' + base_url + uri + ' with request Id: ' + request_id)
r = requests.post(url='https://' + base_url + uri, data=json.dumps(post_body), headers=headers)
# Handle Rate Limiting
if r.status_code == 429:
log.warning('Rate limit hit. sleeping for ' + str(r.headers['X-RateLimit-Reset'] * 1000))
time.sleep(r.headers['X-RateLimit-Reset'] * 1000)
r = requests.post(url='https://' + base_url + uri, data=json.dumps(post_body), headers=headers)
# Handle errors
except Exception as e:
log.error('Unexpected error connecting to API. Exception: ' + str(e))
return 'error'
# Handle errors from API
if r.status_code != 200:
log.error('Request to ' + uri + ' with , request id: ' + request_id + ' returned with status code: ' +
str(r.status_code) + ', response body: ' + r.text)
return 'error'
# Return response body and response headers
return r.content, r.headers
def get_mta_siem_logs(checkpoint_dir, base_url, access_key, secret_key):
uri = "/api/audit/get-siem-logs"
# Set checkpoint file name to store page token
checkpoint_filename = os.path.join(checkpoint_dir, 'get_mta_siem_logs_checkpoint')
# Build post body for request
post_body = dict()
post_body['data'] = [{}]
post_body['data'][0]['type'] = 'MTA'
post_body['data'][0]['compress'] = True
if os.path.exists(checkpoint_filename):
post_body['data'][0]['token'] = read_file(checkpoint_filename)
# Send request to API
resp = post_request(base_url, uri, post_body, access_key, secret_key)
now = datetime.datetime.now().strftime("%a %b %d %H:%M:%S %Y")
# Process response
if resp != 'error':
resp_body = resp[0]
resp_headers = resp[1]
content_type = resp_headers['Content-Type']
# End if response is JSON as there is no log file to download
if content_type == 'application/json':
log.info('No more logs available')
return False
# Process log file
elif content_type == 'application/octet-stream':
file_name = resp_headers['Content-Disposition'].split('=\"')
file_name = file_name[1][:-1]
# Save files to LOG_FILE_PATH
write_file(os.path.join(LOG_FILE_PATH, file_name), resp_body)
# Save mc-siem-token page token to check point directory
write_file(checkpoint_filename, resp_headers['mc-siem-token'])
try:
if syslog_output is True:
for filename in os.listdir(LOG_FILE_PATH):
file_creation_time = time.ctime(os.path.getctime(LOG_FILE_PATH + "/" + filename))
if now < file_creation_time or now == file_creation_time:
log.info('Loading file: ' + filename + ' to output to ' + syslog_server + ':' + str(syslog_port))
with open(file=os.path.join(LOG_FILE_PATH, filename), mode='r', encoding='utf-8') as log_file:
lines = log_file.read().splitlines()
for line in lines:
syslogger.info(line)
log.info('Syslog output completed for file ' + filename)
except Exception as e:
log.error('Unexpected error writing to syslog. Exception: ' + str(e))
# return true to continue loop
return True
else:
# Handle errors
log.error('Unexpected response')
for header in resp_headers:
log.error(header)
return False
def run_script():
# discover base URL
try:
base_url = get_base_url(email_address=EMAIL_ADDRESS)
except Exception as e:
log.error('Error discovering base url for ' + EMAIL_ADDRESS + ' . Exception: ' + str(e))
quit()
# Request log data in a loop until there are no more logs to collect
try:
log.info('Getting MTA log data')
while get_mta_siem_logs(checkpoint_dir=CHK_POINT_DIR, base_url=base_url, access_key=ACCESS_KEY,
secret_key=SECRET_KEY) is True:
log.info('Getting more MTA log files')
except Exception as e:
log.error('Unexpected error getting MTA logs ' + (str(e)))
file_number = len([name for name in os.listdir(LOG_FILE_PATH) if os.path.isfile(name)])
if delete_files or file_number >= log_file_threshold:
for filename in os.listdir(LOG_FILE_PATH):
file_path = os.path.join(LOG_FILE_PATH, filename)
try:
if os.path.isfile(file_path) or os.path.islink(file_path):
os.unlink(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
except Exception as e:
print('Failed to delete %s. Reason: %s' % (file_path, e))
quit()
# Run script
run_script()
It seems like it may potentially be a race condition but I am not sure how to confirm since I can't reproduce it. I notice that SumoLogic has a modified version of this script as well with a different methodology for managing the files/paths. If this script works better than the main sample script above, would anybody be able to explain WHY? I haven't had any issues with it yet.
https://github.com/SumoLogic/sumologic-content/blob/master/MimeCast/SumoLogic-Mimecast-Data-Collection/siem_collection.py
Hello Im new at python and i want remove junk prints in my code (I have indicated the problem in the picture.):
import os
import json
import base64
import sqlite3
import win32crypt
from Crypto.Cipher import AES
import shutil
#ChromeDecoder
print("--------------------| Google Chrome |--------------------")
def get_master_key():
with open(os.environ['USERPROFILE'] + os.sep + r'AppData\Local\Google\Chrome\User Data\Local
State', "r", encoding='utf-8') as f:
local_state = f.read()
local_state = json.loads(local_state)
master_key = base64.b64decode(local_state["os_crypt"]["encrypted_key"])
master_key = master_key[5:] # removing DPAPI
master_key = win32crypt.CryptUnprotectData(master_key, None, None, None, 0)[1]
return master_key
def decrypt_payload(cipher, payload):
return cipher.decrypt(payload)
def generate_cipher(aes_key, iv):
return AES.new(aes_key, AES.MODE_GCM, iv)
def decrypt_password(buff, master_key):
try:
iv = buff[3:15]
payload = buff[15:]
cipher = generate_cipher(master_key, iv)
decrypted_pass = decrypt_payload(cipher, payload)
decrypted_pass = decrypted_pass[:-16].decode() # remove suffix bytes
return decrypted_pass
except Exception as e:
# print("Probably saved password from Chrome version older than v80\n")
# print(str(e))
return "Chrome < 80"
if __name__ == '__main__':
master_key = get_master_key()
login_db = os.environ['USERPROFILE'] + os.sep + r'AppData\Local\Google\Chrome\User
Data\default\Login Data'
shutil.copy2(login_db, "Loginvault.db") #making a temp copy since Login Data DB is locked
while Chrome is running
conn = sqlite3.connect("Loginvault.db")
cursor = conn.cursor()
try:
cursor.execute("SELECT action_url, username_value, password_value FROM logins")
for r in cursor.fetchall():
url = r[0]
username = r[1]
encrypted_password = r[2]
decrypted_password = decrypt_password(encrypted_password, master_key)
print("[+] Password Found !!!" + "\n" +"URL: " + url + "\nUser Name: " + username + "\nPassword: " + decrypted_password + "\n")
except Exception as e:
pass
cursor.close()
conn.close()
try:
os.remove("Loginvault.db")
except Exception as e:
pass
Its works but i have a problem:
enter image description here
I see so much spaces and how i can remove them?
Also is there a way to count found passwords in this format?
print("[+] 100 passwords have been found.")
Sorry for bad English... Thank you.
You can check for empty str also f"text {variable}" is much better for reading.If you want check other values just ad and var != ""
if url != "":
print(f"[+] Password Found !!!\nURL: {url}\nUser Name: {username}\nPassword: {decrypted_password}\n")
2:
before for loop ad count=0
and in loop into if url != "": ad
count += 1
So I'm getting closer, but I can't seem to get this line of code to work just yet: key = manifest["kb"].unwrapKeyForClass(record.protection_class, record.encryption_key[4:])
Attached is the code I have so far, but I still can't wrap my head where to find this information within the backup. Hoping someone can point me in the right direction.
from hashlib import md5
from Crypto.Cipher import AES
from Crypto import Random
import os
import re
import sqlite3
from keystore.keybag import Keybag
from util import readPlist, makedirs
import sys
import plistlib
showinfo = ["Device Name", "Display Name", "Last Backup Date", "IMEI",
"Serial Number", "Product Type", "Product Version", "iTunes Version"]
def extract_backup(backup_path, output_path, password=""):
if not os.path.exists(backup_path + "/Manifest.plist"):
print "Manifest.plist not found"
return
manifest = readPlist(backup_path + "/Manifest.plist")
info = readPlist( backup_path + "/Info.plist")
for i in showinfo:
print i + " : " + unicode(info.get(i, "missing"))
print "Extract backup to %s ? (y/n)" % output_path
if raw_input() == "n":
return
print "Backup is %sencrypted" % (int(not manifest["IsEncrypted"]) * "not ")
if manifest["IsEncrypted"] and password == "":
print "Enter backup password : "
password = raw_input()
if (manifest["IsEncrypted"]):
manifest["password"] = password
manifest["kb"] = Keybag.createWithBackupManifest(manifest, password)
makedirs(output_path)
plistlib.writePlist(manifest, output_path + "/Manifest.plist")
decrypt_backup10(backup_path, output_path, manifest)
def extract_all():
if sys.platform == "win32":
mobilesync = os.environ["APPDATA"] + "/Apple Computer/MobileSync/Backup/"
elif sys.platform == "darwin":
mobilesync = os.environ["HOME"] + "/Library/Application Support/MobileSync/Backup/"
else:
print "Unsupported operating system"
return
print "-" * 60
print "Searching for iTunes backups"
print "-" * 60
for udid in os.listdir(mobilesync):
extract_backup(mobilesync + "/" + udid, udid + "_extract")
def decrypt_backup10(backup_path, output_path, manifest):
connection = sqlite3.connect(backup_path + "/Manifest.db")
try:
connection.row_factory = sqlite3.Row
cursor = connection.cursor()
for record in cursor.execute("SELECT * FROM Files"):
extract_file(backup_path, output_path, record, manifest, kb)
except:
connection.close()
def extract_file(backup_path, output_path, record, manifest):
# read backup file
try:
fileID = record["fileID"]
filename = os.path.join(backup_path, fileID[:2], fileID)
f1 = file(filename, "rb")
print(filename)
except(IOError), e:
#print(e)
print "WARNING: File %s (%s) has not been found" % (filename, record["relativePath"])
return
# write output file
output_path = os.path.join(output_path, fileID[:2], fileID)
ensure_dir_exists(output_path)
print("Writing %s" % output_path)
f2 = file(output_path, 'wb')
aes = None
if manifest["IsEncrypted"] and manifest["kb"]:
key = manifest["kb"].unwrapKeyForClass(record.protection_class, record.encryption_key[4:])
if not key:
warn("Cannot unwrap key")
return
aes = AES.new(key, AES.MODE_CBC, "\x00"*16)
while True:
data = f1.read(8192)
if not data:
break
if aes:
data2 = data = aes.decrypt(data)
f2.write(data)
f1.close()
if aes:
c = data2[-1]
i = ord(c)
if i < 17 and data2.endswith(c*i):
f2.truncate(f2.tell() - i)
else:
warn("Bad padding, last byte = 0x%x !" % i)
f1.close()
f2.close()
def ensure_dir_exists(filename):
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
def main():
extract_backup(input, output, password)
if __name__ == "__main__":
main()
I'm running Flask on Apache with mod_wsgi, every time a file get uploaded, the response time of the server worses, and my memory goes over 100% to swap.
This below is the whole code that handles my upload
#app.route(sApiRoute + '/f/upload', methods=['POST'])
#hasToken
def pbFileUpload(token=None,**kwargs):
def log_time(start_dt=None,start_time=None,doc=None):
end_time = None
if start_time is not None:
end_time = str(time.time()-start_time)
if end_time is not None and end_time > str(0.3):
pblog.warning("Upload with long response time. Start: "+str(start_dt)+ "Duration: "+str(end_time)+" \nUploader: " + token["id_user"] + " \nDoc info: " + str(doc))
import time
start_time = time.time()
start_dt = current_datetime()
# parse path
path = getUploadPath(request.json, None, token)
if not path:
return r404()
# Get the name of the uploaded file
filename = request.json["name"]
filename = filename.replace("$c$","č").replace("$c2$","ć").replace("$z$","ž").replace("$s$","š").replace("$d$","đ").replace("$C$","Č").replace("$C2$","Ć").replace("$Z$","Ž").replace("$S$","Š").replace("$D$","Đ")
rawfile = request.json["data"]
header, data = rawfile.split(',')
# convert data
import base64
recoveredfile = base64.decodebytes(bytes(data, 'UTF-8'))
# write data
import uuid
# leave file name as is
if request.json.get("leaveName"):
generatedfilename = filename
leaveName = True
else:
generatedfilename = str(uuid.uuid1())
leaveName = False
# use of ftp to save files
if 'ftp.' in path.lower():
putFileToFtp(path, filename, generatedfilename, recoveredfile)
# classic directory structure
else:
if not os.path.exists(path):
try:
os.mkdir(path)
except Exception as e:
pblog.warning(str(e),exc_info=1)
return jsonify(data={"status":False,"msg":"Path not found"})
while os.path.exists(os.path.join(path, generatedfilename)) and not leaveName:
generatedfilename = str(uuid.uuid1())
with open(os.path.join(path, generatedfilename), "wb") as out_file:
out_file.write(recoveredfile)
out_file.close()
# send to document system
if request.json.get("dataRec"):
db = DBStore.getDB(token["current_company_id"])
dataRec = request.json.get("dataRec")
dataRec["idext"] = generatedfilename
rtd = DocumentCls.getIdByIdent(dataRec["IdDocumentType"],db)
if rtd and rtd!=[]:
if rtd.get("Id"):
dataRec["IdDocumentType"] = rtd["Id"]
r = DocumentCls.saveDocument(dataRec,request.json.get("username"),db)
log_time(start_dt,start_time,dataRec)
return jsonify(data={"version": r,"generatedfilename":generatedfilename},**kwargs)
else:
log_time(start_dt,start_time,generatedfilename)
return jsonify(data={"filename": filename, "generatedfilename": generatedfilename},**kwargs)
I have to automatically upload folders to an FTP using a Python script. I am able to upload a single file, but not folders with subfolders and files in them. I did a lot of search, but failed. Could some one help me out here? Thanks in advance.
#! /usr/bin/python
import ftplib
s = ftplib.FTP('serverip','usrname','password')
file = '/home/rock/test.txt'
ftppath = '/IT'
filename = "rak"
s.cwd(ftppath)
f = open(file,'rb')
s.storbinary('STOR ' + filename, f)
f.close()
s.quit()
I recently came into this problem and figured out a recursive function to solve it.
import ftplib
import os
server = 'localhost'
username = 'generic_user'
password = 'password'
myFTP = ftplib.FTP(server, username, password)
myPath = r'c:\temp'
def uploadThis(path):
files = os.listdir(path)
os.chdir(path)
for f in files:
if os.path.isfile(path + r'\{}'.format(f)):
fh = open(f, 'rb')
myFTP.storbinary('STOR %s' % f, fh)
fh.close()
elif os.path.isdir(path + r'\{}'.format(f)):
myFTP.mkd(f)
myFTP.cwd(f)
uploadThis(path + r'\{}'.format(f))
myFTP.cwd('..')
os.chdir('..')
uploadThis(myPath) # now call the recursive function
You basically need to use os.walk() to grab those files and transfer them.
Here's a script I wrote for myself to do much of what your asking.
I wrote it a long time ago, so I'd probably do it differently if I wrote it again, but I get a lot of use out of it.
It imports psftplib, which is a wrapper I wrote for the putty sftp.
Feel free to remove these references, or grab the lib at:
http://code.google.com/p/psftplib/source/browse/trunk/psftplib.py
# -*- coding: utf8 -*-
'''This tool will ftp all the files in a given directory to a given location
if the file ftpallcfg.py exists in the directory it will be loaded and the values within it used,
with the current directory used as the source directory.
ftpallcfg.py file contains the following variables.
===========================
server = <server to ftp to>
username = <Username for access to given server>
remote_dir = <remote server directory>
encrypt= True/False
monitor = True/False
walk = True/False
===========================
'''
import ftplib
import os
import getpass
import sys
import time
import socket
import psftplib
__revision__ = 1.11
SLEEP_SECONDS = 1
class FtpAddOns():
PATH_CACHE = []
def __init__(self, ftp_h):
self.ftp_h = ftp_h
def ftp_exists(self, path):
'''path exists check function for ftp handler'''
exists = None
if path not in self.PATH_CACHE:
try:
self.ftp_h.cwd(path)
exists = True
self.PATH_CACHE.append(path)
except ftplib.error_perm, e:
if str(e.args).count('550'):
exists = False
else:
exists = True
return exists
def ftp_mkdirs(self, path, sep='/'):
'''mkdirs function for ftp handler'''
split_path = path.split(sep)
new_dir = ''
for server_dir in split_path:
if server_dir:
new_dir += sep + server_dir
if not self.ftp_exists(new_dir):
try:
print 'Attempting to create directory (%s) ...' % (new_dir),
self.ftp_h.mkd(new_dir)
print 'Done!'
except Exception, e:
print 'ERROR -- %s' % (str(e.args))
def _get_local_files(local_dir, walk=False):
'''Retrieve local files list
result_list == a list of dictionaries with path and mtime keys. ex: {'path':<filepath>,'mtime':<file last modified time>}
ignore_dirs == a list of directories to ignore, should not include the base_dir.
ignore_files == a list of files to ignore.
ignore_file_ext == a list of extentions to ignore.
'''
result_list = []
ignore_dirs = ['CVS', '.svn']
ignore_files = ['.project', '.pydevproject']
ignore_file_ext = ['.pyc']
base_dir = os.path.abspath(local_dir)
for current_dir, dirs, files in os.walk(base_dir):
for this_dir in ignore_dirs:
if this_dir in dirs:
dirs.remove(this_dir)
sub_dir = current_dir.replace(base_dir, '')
if not walk and sub_dir:
break
for this_file in files:
if this_file not in ignore_files and os.path.splitext(this_file)[-1].lower() not in ignore_file_ext:
filepath = os.path.join(current_dir, this_file)
file_monitor_dict = {
'path': filepath,
'mtime': os.path.getmtime(filepath)
}
result_list.append(file_monitor_dict)
return result_list
def monitor_and_ftp(server,
username,
password,
local_dir,
remote_dir,
encrypt=False,
walk=False):
'''Monitor local files and when an update is found connect and upload'''
print 'Monitoring changes in (%s).' % (os.path.abspath(local_dir))
print '(Use ctrl-c to exit)'
last_files_list = _get_local_files(local_dir)
while True:
try:
time.sleep(SLEEP_SECONDS)
latest_files_list = _get_local_files(local_dir)
files_to_update = []
for idx in xrange(len(latest_files_list)):
if idx < len(last_files_list):
# compare last modified times
if latest_files_list[idx]['mtime'] > last_files_list[idx]['mtime']:
files_to_update.append(latest_files_list[idx])
else:
# add the file to the list (new file)
files_to_update.append(latest_files_list[idx])
if files_to_update:
print
print 'Detected NEW or CHANGED file(s), attempting to send ...'
print
is_success = upload_all(server,
username,
password,
local_dir,
remote_dir,
files_to_update,
encrypt,
walk)
if not is_success:
break
else:
print '.',
last_files_list = latest_files_list[:] # copy the list to hold
except KeyboardInterrupt:
print
print 'Exiting.'
break
def upload_all(server,
username,
password,
base_local_dir,
base_remote_dir,
files_to_update=None,
encrypt=False,
walk=False):
'''Upload all files in a given directory to the given remote directory'''
continue_on = False
login_ok = False
server_connect_ok = False
base_local_dir = os.path.abspath(base_local_dir)
base_remote_dir = os.path.normpath(base_remote_dir)
if files_to_update:
local_files = files_to_update
else:
local_files = _get_local_files(base_local_dir, walk)
if local_files:
if not encrypt: # Use standard FTP
ftp_h = ftplib.FTP()
else: # Use sftp
ftp_h = psftplib.SFTP()
try:
ftp_h.connect(server)
server_connect_ok = True
except socket.gaierror, e:
print 'ERROR -- Could not connect to (%s): %s' % (server, str(e.args))
except IOError, e:
print 'ERROR -- File not found: %s' % (str(e.args))
except socket.error, e:
print 'ERROR -- Could not connect to (%s): %s' % (server, str(e.args))
ftp_path_tools = FtpAddOns(ftp_h)
if server_connect_ok:
try:
ftp_h.login(username,password)
print 'Logged into (%s) as (%s)' % (server, username)
login_ok = True
except ftplib.error_perm, e:
print 'ERROR -- Check Username/Password: %s' % (str(e.args))
except psftplib.ProcessTimeout, e:
print 'ERROR -- Check Username/Password (timeout): %s' % (str(e.args))
if login_ok:
for file_info in local_files:
filepath = file_info['path']
path, filename = os.path.split(filepath)
remote_sub_path = path.replace(base_local_dir, '')
remote_path = path.replace(base_local_dir, base_remote_dir)
remote_path = remote_path.replace('\\', '/') # Convert to unix style
if not ftp_path_tools.ftp_exists(remote_path):
ftp_path_tools.ftp_mkdirs(remote_path)
# Change to directory
try:
ftp_h.cwd(remote_path)
continue_on = True
except ftplib.error_perm, e:
print 'ERROR -- %s' % (str(e.args))
except psftplib.PsFtpInvalidCommand, e:
print 'ERROR -- %s' % (str(e.args))
if continue_on:
if os.path.exists(filepath):
f_h = open(filepath,'rb')
filename = os.path.split(f_h.name)[-1]
display_filename = os.path.join(remote_sub_path, filename)
display_filename = display_filename.replace('\\', '/')
print 'Sending (%s) ...' % (display_filename),
send_cmd = 'STOR %s' % (filename)
try:
ftp_h.storbinary(send_cmd, f_h)
f_h.close()
print 'Done!'
except Exception, e:
print 'ERROR!'
print str(e.args)
print
else:
print "WARNING -- File no longer exists, (%s)!" % (filepath)
ftp_h.quit()
print 'Closing Connection'
else:
print 'ERROR -- No files found in (%s)' % (base_local_dir)
return continue_on
if __name__ == '__main__':
import optparse
default_config_file = u'ftpallcfg.py'
# Create parser, and configure command line options to parse
parser = optparse.OptionParser()
parser.add_option("-l", "--local_dir",
dest="local_dir",
help="Local Directory (Defaults to CWD)",
default='.')
parser.add_option("-r", "--remote_dir",
dest="remote_dir",
help="[REQUIRED] Target Remote directory",
default=None)
parser.add_option("-u", "--username",
dest="username",
help="[REQUIRED] username",
default=None)
parser.add_option("-s","--server",
dest="server",
help="[REQUIRED] Server Address",
default=None)
parser.add_option("-e", "--encrypt",
action="store_true",
dest="encrypt",
help="Use sftp",
default=False)
parser.add_option("-m",
action="store_true",
dest="monitor",
help="Keep process open and monitor changes",
default=False)
parser.add_option("-w",
action="store_true",
dest="walkdir",
help="Walk sub directories of the given directory to find files to send.",
default=False)
(options,args) = parser.parse_args()
if (options.username and options.server and options.remote_dir) or \
os.path.exists(default_config_file):
local_dir = options.local_dir
if os.path.exists(default_config_file):
sys.path.append('.')
import ftpallcfg
try:
server = ftpallcfg.server
username = ftpallcfg.username
remote_dir = ftpallcfg.remote_dir
encrypt = ftpallcfg.encrypt
monitor = ftpallcfg.monitor
walk = ftpallcfg.walk
except AttributeError, e:
print "ERROR --", str(e.args)
print
print 'Value(s) missing in %s file! The following values MUST be included:' % (default_config_file)
print '================================'
print 'server = <server to ftp to>'
print 'username = <Username for access to given server>'
print 'remote_dir = <remote server directory>'
print 'encrypt= True/False'
print 'monitor = True/False'
print 'walk == True/False'
print '================================'
sys.exit()
else:
server = options.server
username = options.username
remote_dir = options.remote_dir
encrypt = options.encrypt
monitor = options.monitor
walk = options.walkdir
# get the user password
prompt = 'Password (%s#%s): ' % (username, server)
if os.isatty(sys.stdin.fileno()):
p = getpass.getpass(prompt)
else:
#p = sys.stdin.readline().rstrip()
p = raw_input(prompt).rstrip()
if options.encrypt:
print '>> Using sftp for secure transfers <<'
print
if monitor:
try:
monitor_and_ftp(server,username,p,local_dir, remote_dir, encrypt, walk)
except KeyboardInterrupt:
print 'Exiting...'
else:
try:
upload_all(server, username, p, local_dir, remote_dir, [], encrypt, walk)
except KeyboardInterrupt:
print 'Exiting...'
else:
print 'ERROR -- Required option not given!'
print __revision__
print __doc__
print
parser.print_help()
EDIT 20/12/2017:
I have written a project in GitHub for this purpose. Click for details!
There are good answers above but i also want to add a good one using ftputil package. If you need to upload files from local directory to ftp directory, you can use this recursive function:
def upload_dir(localDir, ftpDir):
list = os.listdir(localDir)
for fname in list:
if os.path.isdir(localDir + fname):
if(ftp_host.path.exists(ftpDir + fname) != True):
ftp_host.mkdir(ftpDir + fname)
print(ftpDir + fname + " is created.")
upload_dir(localDir + fname + "/", ftpDir + fname + "/")
else:
if(ftp_host.upload_if_newer(localDir + fname, ftpDir + fname)):
print(ftpDir + fname + " is uploaded.")
else:
print(localDir + fname + " has already been uploaded.")
If you decide to use this function, you have to connect ftp using ftputil package. For this, you can use this snippet:
with ftputil.FTPHost("ftp_host", "ftp_username", "ftp_password") as ftp_host:
So, we're almost done. The last thing is usage of the function for beginners like me:
local_dir = "D:/Projects/.../"
ftp_dir = "/.../../"
upload_dir(local_dir, ftp_dir)
The most important thing is "/" character at the end of paths. You need to put it at the end. Finally, i want to share entire code:
with ftputil.FTPHost("ftp_host", "ftp_username", "ftp_password") as ftp_host:
def upload_dir(localDir, ftpDir):
list = os.listdir(localDir)
for fname in list:
if os.path.isdir(localDir + fname):
if(ftp_host.path.exists(ftpDir + fname) != True):
ftp_host.mkdir(ftpDir + fname)
print(ftpDir + fname + " is created.")
upload_dir(localDir + fname + "/", ftpDir + fname + "/")
else:
if(ftp_host.upload_if_newer(localDir + fname, ftpDir + fname)):
print(ftpDir + fname + " is uploaded.")
else:
print(localDir + fname + " has already been uploaded.")
local_dir = "D:/Projects/.../"
ftp_dir = "/.../../"
upload_dir(local_dir, ftp_dir)
Maybe you try ftpsync.py. If this one doesn't helps, try google search on python ftpsync and you get a lot of answers.
using ftputil:
import os
import ftputil
import ftputil.session
def upload_dir(root):
root = unicode(root, 'utf-8')
for dir_name, _, dir_files in os.walk(root):
local = os.path.join(os.curdir, dir_name)
remote = ftp_host.path.join(ftp_host.curdir, dir_name)
if not ftp_host.path.exists(remote):
print 'mkdir:', local, '->', remote
ftp_host.mkdir(remote)
for f in dir_files:
local_f = os.path.join(local, f)
remote_f = ftp_host.path.join(remote, f)
print 'upload:', local_f, '->', remote_f
ftp_host.upload(local_f, remote_f)
sf = ftputil.session.session_factory(use_passive_mode=True)
with ftputil.FTPHost('HOST', 'USER', 'PASS', session_factory=sf) as ftp_host:
upload_dir('DIR')
It is easy to use lftp to upload folders to an FTP. I use this in my Python script to move folders to FTP
Python script:
#! /usr/bin/python
import subprocess
subprocess.call(["bash", ftp_script, username, password, ftp, folder_to_move, src,folder_name_in_destination])
ftp_script:
lftp -u $1,$2 $3 <<EOF
mkdir $4
lcd $5
cd $6
mirror --reverse
EOF
Yet another answer using ftputil. And pathlib for figuring out the local path.
I've compiled and improved some of the above answers.
So thank you for the inspiration!
import os
import ftplib
import ftputil
from pathlib import Path
def ftp_upload(ftp_host, src_path, dst_path=None):
if dst_path is None:
ftp_host.path.join(os.curdir)
# Make sure that local path is a directory
if src_path.is_dir():
# Make sure that the root path exists
if ftp_host.path.exists(dst_path) is False:
# Inform user
print(f'mkdir: {src_path} -> {dst_path}')
# Create folder
ftp_host.mkdir(dst_path)
# Parse content of the root folder
for src_child in src_path.iterdir():
if src_child.is_dir():
# Use recursion for sub folders/files
dst_child = ftp_host.path.join(dst_path, src_child.stem)
ftp_upload(ftp_host, src_child, dst_child)
else:
# Copy sub files
dst_child = ftp_host.path.join(dst_path, src_child.name)
# Inform user
print(f'upload: {src_child} -> {dst_child}')
# Perform copy (upload)
ftp_host.upload(src_child, dst_child)
# ftp_host.upload_if_newer(src_child, dst_child)
else:
# Inform user
print(f"src_path: '{src_path}' must be an existing directory!")
if __name__ == '__main__':
# FTP settings
user = "ftp_user"
password = "ftp_password"
host = "localhost"
port = 2121
# Path settings
src_path = Path(__file__).parent.resolve() / "upload_dir"
dst_dir = "upload_dir"
# Inform user
print(f"Establishing FTP session as '{user}'")
# Establish FTP session
ftplib.FTP.port = port
with ftputil.FTPHost(host, user, password) as ftp_host:
# Perform recursive FTP upload
dst_path = ftp_host.path.join(os.curdir, dst_dir)
ftp_upload(ftp_host, src_path, dst_path)