Develop Reference

log setup with formatting and logrotate

Below program I am trying to setup the logs with my required formatting using basicConfig and use RotatingFileHandler to rotate the logs.
But getting the Error:
Traceback (most recent call last):
File "C:\Python27\lib\logging\handlers.py", line 77, in emit
self.doRollover()
File "C:\Python27\lib\logging\handlers.py", line 142, in doRollover
os.rename(self.baseFilename, dfn)
WindowsError: [Error 32] The process cannot access the file because it is being used by another process
Logged from file log_rotate.py, line 53
Could not figure out where is the problem, new to python.
Can someone please point me in right direction.
import os
import logging
import time
import string
from ctypes import windll
from logging.handlers import RotatingFileHandler
LOG_FILE = 'C:\\temp\\debug.log'
def get_drives():
drives = []
bitmask = windll.kernel32.GetLogicalDrives()
for letter in string.ascii_uppercase:
if bitmask & 1:
drives.append(letter)
bitmask >>= 1
return drives
def create_temp_dir():
drives = []
drives = get_drives()
temp_dir = drives[0]+':\\temp\\'
if not os.path.exists(temp_dir):
print ( " creating temp for logs and etc ", temp_dir )
os.makedirs(temp_dir)
return temp_dir
def make_log_setup():
global LOG_FILE
temp_dir = create_temp_dir()
log_file = temp_dir+'debug.log'
date_strftime_format = "%d-%b-%y %H:%M:%S"
message_format='%(asctime)s %(levelname)s %(module)s - %(funcName)s: %(message)s'
logging.basicConfig(filename = log_file, format = message_format, datefmt = date_strftime_format,level=logging.DEBUG)
def create_rotating_log(path):
"""
Creates a rotating log
"""
logger = logging.getLogger("Rotating Log")
logger.setLevel(logging.DEBUG)
# add a rotating handler
handler = RotatingFileHandler(path, maxBytes=20,
backupCount=5)
logger.addHandler(handler)
for i in range(6):
logger.info("This is test log line %s" % i)
time.sleep(1.5)
if __name__ == "__main__":
#log_file = "test.log"
make_log_setup()
create_rotating_log(LOG_FILE)

PermissionError: [WinError 5] Access is denied OS.remove

i am trying to remove the folder variable "name" if its been in the folder longer than X amount of time. Can i run this script in admin mode without having to "right click" and run as admin? If i try to automate this script i would need something to that nature. I try using the os.remove function but i get an error below:
Error
PermissionError: [WinError 5] Access is denied:
Code:
for root, folders, files in os.walk('\\\MYDATA\\user$\\test\\Documents\\chris2020\\test.fof'):
for name in folders:
datetimeFormat = '%Y-%m-%d %H:%M:%S.%f'
filedate = str(datetime.fromtimestamp(os.path.getmtime(os.path.join(root, name))))
now_time = str(datetime.now())
now_time = datetime.strptime(now_time, datetimeFormat)
filetime = datetime.strptime(filedate, datetimeFormat)
difference = now_time-filetime
if difference > timedelta(days=2):
print(filetime)
print(difference)
print('Hi')
# os.remove('\\\MYDATA\\user$\\test\\Documents\\chris2020\\test.fof\\' + name)
shutil.rmtree('\\\MYDATA\\user$\\test\\Documents\\chris2020\\test.fof\\' + name)
file_times = os.path.join("\\\MYDATA\\user$\\test\\Documents\\chris2020\\test.fof\\", name), ": ", str(
difference)
file_times_final.append(file_times[0] + file_times[1] + file_times[2])
else:
print("None")
break

Assuming the issue is that Python is not elevated, the solution provided here might be useful.
To run an external command from within Python, this solution could be suitable:
#!python
# coding: utf-8
import sys
import ctypes
def run_as_admin(argv=None, debug=False):
shell32 = ctypes.windll.shell32
if argv is None and shell32.IsUserAnAdmin():
return True
if argv is None:
argv = sys.argv
if hasattr(sys, '_MEIPASS'):
# Support pyinstaller wrapped program.
arguments = map(unicode, argv[1:])
else:
arguments = map(unicode, argv)
argument_line = u' '.join(arguments)
executable = unicode(sys.executable)
if debug:
print 'Command line: ', executable, argument_line
ret = shell32.ShellExecuteW(None, u"runas", executable, argument_line, None, 1)
if int(ret) <= 32:
return False
return None
if __name__ == '__main__':
ret = run_as_admin()
if ret is True:
print 'I have admin privilege.'
raw_input('Press ENTER to exit.')
elif ret is None:
print 'I am elevating to admin privilege.'
raw_input('Press ENTER to exit.')
else:
print 'Error(ret=%d): cannot elevate privilege.' % (ret, )
The key lines seem to be
import ctypes
shell32 = ctypes.windll.shell32
shell32.ShellExecuteW(None, u"runas", executable, argument_line, None, 1)

Getting a SyntaxError on the last line of code in a Unison Merge script

Hey brilliant minds out there,
I'm trying to get this Python script that I found working and can't figure out if it's just me or if the original poster didn't get the code right. This script is supposed to enable automatic conflict resolution in Unison by utilising the merge command in the prefs file by taking the two conflicting files and duplicating one of them with a datestamp in the filename. The original posing is here but there wasn't any indentation so I've had to go through and do it manually just by watching the errors that pop up. The error that I can't seem to get around now is
File "/bin/unison_merge.py", line 5, in <module>
PATH, CURRENT1, CURRENT2, NEW = sys.argv[1:]
ValueError: need more than 0 values to unpack
I'm hoping that someone out there will be able to help me out.
I've included the entire script below in the hope that it will help and that any other errors will be noticed :).
#!/usr/bin/env python2.7
import sys, os, datetime, os, filecmp
PATH, CURRENT1, CURRENT2, NEW = sys.argv[1:]
# see http://www.cis.upenn.edu/~bcpierce/unison/download/releases/stable/unison-manual.html#merge
promote_remote = False
backup_file_color = "red"
def is_suffix(a, b): return b[-len(a):] == a
def merge(PATH, CURRENT1, CURRENT2, NEW, promote_remote):
# CURRENT1 is copy of local, CURRENT2 is copy of remote
if filecmp.cmp(CURRENT1, CURRENT2, shallow = False):
# special case -- files have same contents
# not a real conflict. just use local copy, no backup
print "merge of identical files"
os.link(CURRENT1, NEW)
return
# PATH is relative to unison root.
# We need to know absolute path.
# We get it, assuming CURRENT1 is an absolute path
# referring to a file in the same subdirectory as PATH.
assert CURRENT1[0] == '/', "CURRENT1 (%s) is not absolute path" % CURRENT1
PATH_dir, PATH_tail = os.path.split(PATH)
ABS_dir = os.path.dirname(CURRENT1)
assert is_suffix(PATH_dir, ABS_dir), "%s not suffix of %s!" % (PATH_dir, ABS_dir)
ABS_PATH = os.path.join(ABS_dir, PATH_tail)
timestamp = datetime.datetime.now().strftime("%y%m%d_%H%M")
(root, ext) = os.path.splitext(PATH_tail)
for counter in range(100):
counter = " %d" % counter if counter else ""
filename = "%s #%s%s%s" % (root, timestamp, counter, ext)
BACKUP = os.path.join(ABS_dir, filename)
if not os.path.exists(BACKUP): break
else:
assert False, "too many existing backups %s" % BACKUP
# promote_remote = False
# seems to retain file props, saving update in next sync?
print "CONFLICT:", ABS_PATH
if promote_remote:
# resolve conflict by using remote copy, while backing up local to
BACKUP
CURRENT1, CURRENT2 = CURRENT2, CURRENT1
print "CONFLICT remote saved as", filename
else:
print "CONFLICT local saved as", filename
assert os.path.isfile(CURRENT1)
assert not os.path.exists(NEW)
assert not os.path.exists(BACKUP)
os.link(CURRENT1, BACKUP)
os.link(CURRENT2, NEW)
if backup_file_color and backup_file_color != 'none':
mac_color_file(BACKUP, backup_file_color)
# note: coloring the tmp file NEW is useless - not propagated
# coloring the current file ABS_PATH causes UNISON to complain
# chmod -w BACKUP
# os.chmod(BACKUP, stat.S_IRUSR)
# just for coloring file in mac Finder
def mac_color_file(file, color):
if not os.path.exists("/usr/bin/osascript"): return
color_map = {
"none":0,
"orange":1,
"red":2,
"yellow":3,
"blue":4,
"purple":5,
"green":6,
"gray":7,
}
assert color in color_map
assert file[0] == '/', 'absolute path required'
assert os.path.exists(file)
#see http://stackoverflow.com/questions/2435580/tagging-files-with-colors-in-os-x-finder-from-shell-scripts
#osascript -e "tell application \"Finder\" to set label index of alias POSIX
file ("$filename\" to $label")
cmd = '''/usr/bin/osascript -e 'tell application "Finder" to set label index of alias POSIX file "%s" to %d' > /dev/null ''' % (file, color_map[color])
try:
retcode = subprocess.call(cmd, shell=True)
if retcode < 0:
print >>sys.stderr, "mac_color_file child was terminated by signal", retcode
elif retcode > 0:
print >>sys.stderr, "mac_color_file child returned", retcode
except OSError, e:
print >>sys.stderr, "mac_color_file child failed:", e
### main ###
merge(PATH, CURRENT1, CURRENT2, NEW, promote_remote)

The try block doesn't have an accompanying except.
This tutorial page clarifies the use of the try..except structure:
http://docs.python.org/tutorial/errors.html#handling-exceptions

This is just a half hearted attempt.
The indentation were really off and I am not sure what the program is trying to do.
I hope that there is a single big function to be kicked off.
To run the program from command line, you must provide command line arguments.
python test.py x y c v
In this case:
PATH, CURRENT1, CURRENT2, NEW = 'x', 'y', 'c', 'z'
Program
#!/usr/bin/env python2.7
import sys, os, datetime, os, filecmp
print sys.argv
PATH, CURRENT1, CURRENT2, NEW = sys.argv[1:]
# see http://www.cis.upenn.edu/~bcpierce/unison/download/releases/stable/unison-manual.html#merge
promote_remote = False
backup_file_color = "red"
def is_suffix(a, b):
return b[-len(a):] == a
def merge(PATH, CURRENT1, CURRENT2, NEW, promote_remote):
# CURRENT1 is copy of local, CURRENT2 is copy of remote
if filecmp.cmp(CURRENT1, CURRENT2, shallow = False):
# special case -- files have same contents
# not a real conflict. just use local copy, no backup
print "merge of identical files"
os.link(CURRENT1, NEW)
return
# PATH is relative to unison root.
# We need to know absolute path.
# We get it, assuming CURRENT1 is an absolute path
# referring to a file in the same subdirectory as PATH.
assert CURRENT1[0] == '/', "CURRENT1 (%s) is not absolute path" % CURRENT1
PATH_dir, PATH_tail = os.path.split(PATH)
ABS_dir = os.path.dirname(CURRENT1)
assert is_suffix(PATH_dir, ABS_dir), "%s not suffix of %s!" % (PATH_dir, ABS_dir)
ABS_PATH = os.path.join(ABS_dir, PATH_tail)
timestamp = datetime.datetime.now().strftime("%y%m%d_%H%M")
(root, ext) = os.path.splitext(PATH_tail)
for counter in range(100):
counter = " %d" % counter if counter else ""
filename = "%s #%s%s%s" % (root, timestamp, counter, ext)
BACKUP = os.path.join(ABS_dir, filename)
if not os.path.exists(BACKUP):
break
else:
assert False, "too many existing backups %s" % BACKUP
# promote_remote = False
# seems to retain file props, saving update in next sync?
print "CONFLICT:", ABS_PATH
if promote_remote:
# resolve conflict by using remote copy, while backing up local to BACKUP
CURRENT1, CURRENT2 = CURRENT2, CURRENT1
print "CONFLICT remote saved as", filename
else:
print "CONFLICT local saved as", filename
assert os.path.isfile(CURRENT1)
assert not os.path.exists(NEW)
assert not os.path.exists(BACKUP)
os.link(CURRENT1, BACKUP)
os.link(CURRENT2, NEW)
if backup_file_color and backup_file_color != 'none':
mac_color_file(BACKUP, backup_file_color)
# note: coloring the tmp file NEW is useless - not propagated
# coloring the current file ABS_PATH causes UNISON to complain
# chmod -w BACKUP
# os.chmod(BACKUP, stat.S_IRUSR)
# just for coloring file in mac Finder
def mac_color_file(file, color):
if not os.path.exists("/usr/bin/osascript"):
return
color_map = {
"none":0,
"orange":1,
"red":2,
"yellow":3,
"blue":4,
"purple":5,
"green":6,
"gray":7,
}
assert color in color_map
assert file[0] == '/', 'absolute path required'
assert os.path.exists(file)
#see http://stackoverflow.com/questions/2435580/tagging-files-with-colors-in-os-x-finder-from-shell-scripts
#osascript -e "tell application \"Finder\" to set label index of alias POSIX
file ("$filename\" to $label")
cmd = '''/usr/bin/osascript -e 'tell application "Finder" to set label index of alias POSIX file "%s" to %d' > /dev/null ''' % (file, color_map[color])
try:
retcode = subprocess.call(cmd, shell=True)
if retcode < 0:
print >>sys.stderr, "mac_color_file child was terminated by signal", retcode
elif retcode > 0:
print >>sys.stderr, "mac_color_file child returned", retcode
except:
pass
if __file__ == '__main__':
merge(PATH, CURRENT1, CURRENT2, NEW, promote_remote)

Here's the current script. Note that it expects to be invoked with four arguments as described in the unison documentation. To trigger it, add a line like the following to your unison .prf file:
merge = Name * -> /Users/neal/Bin/daemons/unison_merge.py 'PATH' CURRENT1 CURRENT2 NEW
The single quotes are (were?) necessary to work around a bug in unison.
Here's the script:
#!/usr/bin/env python2.7
# see http://www.cis.upenn.edu/~bcpierce/unison/download/releases/stable/unison-manual.html#merge
import sys, os, datetime, os, stat, subprocess, filecmp
# log = open("/Users/neal/unison_merge.log", "a")
# sys.stdout = log
# sys.stderr = log
# work around bug in unison adds quotes incorrectly
# something to do with blanks in filenames
def clean(x):
if len(x) < 3: return x
if x[0] == "'": x = x[1:]
if x[-1] == "'": x = x[:-1]
return x
sys.argv = [clean(x) for x in sys.argv]
try:
PATH, CURRENT1, CURRENT2, NEW = sys.argv[1:]
except:
print "usage: ", " ".join("[%s]" % x for x in sys.argv)
raise
backup_file_color = 'red' # for mac
promote_remote = False
def is_suffix(a, b): return b[-len(a):] == a
def mac_color_file(file, color):
if not os.path.exists("/usr/bin/osascript"): return
color_map = {
"none":0,
"orange":1,
"red":2,
"yellow":3,
"blue":4,
"purple":5,
"green":6,
"gray":7,
}
assert color in color_map
assert file[0] == '/', 'absolute path required'
assert os.path.exists(file)
#see http://stackoverflow.com/questions/2435580/tagging-files-with-colors-in-os-x-finder-from-shell-scripts
#osascript -e "tell application \"Finder\" to set label index of alias POSIX file \"$filename\" to $label"
cmd = '''/usr/bin/osascript -e 'tell application "Finder" to set label index of alias POSIX file "%s" to %d' > /dev/null ''' % (file, color_map[color])
try:
retcode = subprocess.call(cmd, shell=True)
if retcode < 0:
print >>sys.stderr, "mac_color_file child was terminated by signal", -retcode
elif retcode > 0:
print >>sys.stderr, "mac_color_file child returned", retcode
except OSError, e:
print >>sys.stderr, "mac_color_file child failed:", e
def merge(PATH, CURRENT1, CURRENT2, NEW, promote_remote):
# CURRENT1 is copy of local, CURRENT2 is copy of remote
# PATH is relative to unison root.
# We need to know absolute path.
# We get it, assuming CURRENT1 is an absolute path
# referring to a file in the same subdirectory as PATH.
if filecmp.cmp(CURRENT1, CURRENT2, shallow = False):
# special case -- files have same contents
# not a real conflict. just use local copy, no backup
print "merge of identical files"
os.link(CURRENT1, NEW)
return
assert CURRENT1[0] == '/', "CURRENT1 (%s) is not absolute path" % CURRENT1
PATH_dir, PATH_tail = os.path.split(PATH)
ABS_dir = os.path.dirname(CURRENT1)
assert is_suffix(PATH_dir, ABS_dir), "%s not suffix of %s!" % (PATH_dir, ABS_dir)
ABS_PATH = os.path.join(ABS_dir, PATH_tail)
timestamp = datetime.datetime.now().strftime("%y%m%d_%H%M")
(root, ext) = os.path.splitext(PATH_tail)
for counter in range(100):
counter = " %d" % counter if counter else ""
filename = "%s #%s%s%s" % (root, timestamp, counter, ext)
BACKUP = os.path.join(ABS_dir, filename)
if not os.path.exists(BACKUP): break
else:
assert False, "too many existing backups %s" % BACKUP
# promote_remote = False
# seems to retain file props, saving update in next sync?
print "CONFLICT:", ABS_PATH
if promote_remote:
# resolve conflict by using remote copy, while backing up local to BACKUP
CURRENT1, CURRENT2 = CURRENT2, CURRENT1
print "CONFLICT remote saved as", filename
else:
print "CONFLICT local saved as", filename
assert os.path.isfile(CURRENT1)
assert not os.path.exists(NEW)
assert not os.path.exists(BACKUP)
os.link(CURRENT1, BACKUP)
os.link(CURRENT2, NEW)
if backup_file_color and backup_file_color != 'none':
mac_color_file(BACKUP, backup_file_color)
# note: coloring the tmp file NEW is useless - not propagated
# coloring the current file ABS_PATH causes UNISON to complain
# chmod -w BACKUP
# os.chmod(BACKUP, stat.S_IRUSR)
### main ###
try:
merge(PATH, CURRENT1, CURRENT2, NEW, promote_remote)
except Exception as e:
print >>sys.stderr, "ERROR in unison_merge.py"
print >>sys.stderr, "ERROR:", str(e)

Upload folders from local system to FTP using Python script

I have to automatically upload folders to an FTP using a Python script. I am able to upload a single file, but not folders with subfolders and files in them. I did a lot of search, but failed. Could some one help me out here? Thanks in advance.
#! /usr/bin/python
import ftplib
s = ftplib.FTP('serverip','usrname','password')
file = '/home/rock/test.txt'
ftppath = '/IT'
filename = "rak"
s.cwd(ftppath)
f = open(file,'rb')
s.storbinary('STOR ' + filename, f)
f.close()
s.quit()

I recently came into this problem and figured out a recursive function to solve it.
import ftplib
import os
server = 'localhost'
username = 'generic_user'
password = 'password'
myFTP = ftplib.FTP(server, username, password)
myPath = r'c:\temp'
def uploadThis(path):
files = os.listdir(path)
os.chdir(path)
for f in files:
if os.path.isfile(path + r'\{}'.format(f)):
fh = open(f, 'rb')
myFTP.storbinary('STOR %s' % f, fh)
fh.close()
elif os.path.isdir(path + r'\{}'.format(f)):
myFTP.mkd(f)
myFTP.cwd(f)
uploadThis(path + r'\{}'.format(f))
myFTP.cwd('..')
os.chdir('..')
uploadThis(myPath) # now call the recursive function

You basically need to use os.walk() to grab those files and transfer them.
Here's a script I wrote for myself to do much of what your asking.
I wrote it a long time ago, so I'd probably do it differently if I wrote it again, but I get a lot of use out of it.
It imports psftplib, which is a wrapper I wrote for the putty sftp.
Feel free to remove these references, or grab the lib at:
http://code.google.com/p/psftplib/source/browse/trunk/psftplib.py
# -*- coding: utf8 -*-
'''This tool will ftp all the files in a given directory to a given location
if the file ftpallcfg.py exists in the directory it will be loaded and the values within it used,
with the current directory used as the source directory.
ftpallcfg.py file contains the following variables.
===========================
server = <server to ftp to>
username = <Username for access to given server>
remote_dir = <remote server directory>
encrypt= True/False
monitor = True/False
walk = True/False
===========================
'''
import ftplib
import os
import getpass
import sys
import time
import socket
import psftplib
__revision__ = 1.11
SLEEP_SECONDS = 1
class FtpAddOns():
PATH_CACHE = []
def __init__(self, ftp_h):
self.ftp_h = ftp_h
def ftp_exists(self, path):
'''path exists check function for ftp handler'''
exists = None
if path not in self.PATH_CACHE:
try:
self.ftp_h.cwd(path)
exists = True
self.PATH_CACHE.append(path)
except ftplib.error_perm, e:
if str(e.args).count('550'):
exists = False
else:
exists = True
return exists
def ftp_mkdirs(self, path, sep='/'):
'''mkdirs function for ftp handler'''
split_path = path.split(sep)
new_dir = ''
for server_dir in split_path:
if server_dir:
new_dir += sep + server_dir
if not self.ftp_exists(new_dir):
try:
print 'Attempting to create directory (%s) ...' % (new_dir),
self.ftp_h.mkd(new_dir)
print 'Done!'
except Exception, e:
print 'ERROR -- %s' % (str(e.args))
def _get_local_files(local_dir, walk=False):
'''Retrieve local files list
result_list == a list of dictionaries with path and mtime keys. ex: {'path':<filepath>,'mtime':<file last modified time>}
ignore_dirs == a list of directories to ignore, should not include the base_dir.
ignore_files == a list of files to ignore.
ignore_file_ext == a list of extentions to ignore.
'''
result_list = []
ignore_dirs = ['CVS', '.svn']
ignore_files = ['.project', '.pydevproject']
ignore_file_ext = ['.pyc']
base_dir = os.path.abspath(local_dir)
for current_dir, dirs, files in os.walk(base_dir):
for this_dir in ignore_dirs:
if this_dir in dirs:
dirs.remove(this_dir)
sub_dir = current_dir.replace(base_dir, '')
if not walk and sub_dir:
break
for this_file in files:
if this_file not in ignore_files and os.path.splitext(this_file)[-1].lower() not in ignore_file_ext:
filepath = os.path.join(current_dir, this_file)
file_monitor_dict = {
'path': filepath,
'mtime': os.path.getmtime(filepath)
}
result_list.append(file_monitor_dict)
return result_list
def monitor_and_ftp(server,
username,
password,
local_dir,
remote_dir,
encrypt=False,
walk=False):
'''Monitor local files and when an update is found connect and upload'''
print 'Monitoring changes in (%s).' % (os.path.abspath(local_dir))
print '(Use ctrl-c to exit)'
last_files_list = _get_local_files(local_dir)
while True:
try:
time.sleep(SLEEP_SECONDS)
latest_files_list = _get_local_files(local_dir)
files_to_update = []
for idx in xrange(len(latest_files_list)):
if idx < len(last_files_list):
# compare last modified times
if latest_files_list[idx]['mtime'] > last_files_list[idx]['mtime']:
files_to_update.append(latest_files_list[idx])
else:
# add the file to the list (new file)
files_to_update.append(latest_files_list[idx])
if files_to_update:
print
print 'Detected NEW or CHANGED file(s), attempting to send ...'
print
is_success = upload_all(server,
username,
password,
local_dir,
remote_dir,
files_to_update,
encrypt,
walk)
if not is_success:
break
else:
print '.',
last_files_list = latest_files_list[:] # copy the list to hold
except KeyboardInterrupt:
print
print 'Exiting.'
break
def upload_all(server,
username,
password,
base_local_dir,
base_remote_dir,
files_to_update=None,
encrypt=False,
walk=False):
'''Upload all files in a given directory to the given remote directory'''
continue_on = False
login_ok = False
server_connect_ok = False
base_local_dir = os.path.abspath(base_local_dir)
base_remote_dir = os.path.normpath(base_remote_dir)
if files_to_update:
local_files = files_to_update
else:
local_files = _get_local_files(base_local_dir, walk)
if local_files:
if not encrypt: # Use standard FTP
ftp_h = ftplib.FTP()
else: # Use sftp
ftp_h = psftplib.SFTP()
try:
ftp_h.connect(server)
server_connect_ok = True
except socket.gaierror, e:
print 'ERROR -- Could not connect to (%s): %s' % (server, str(e.args))
except IOError, e:
print 'ERROR -- File not found: %s' % (str(e.args))
except socket.error, e:
print 'ERROR -- Could not connect to (%s): %s' % (server, str(e.args))
ftp_path_tools = FtpAddOns(ftp_h)
if server_connect_ok:
try:
ftp_h.login(username,password)
print 'Logged into (%s) as (%s)' % (server, username)
login_ok = True
except ftplib.error_perm, e:
print 'ERROR -- Check Username/Password: %s' % (str(e.args))
except psftplib.ProcessTimeout, e:
print 'ERROR -- Check Username/Password (timeout): %s' % (str(e.args))
if login_ok:
for file_info in local_files:
filepath = file_info['path']
path, filename = os.path.split(filepath)
remote_sub_path = path.replace(base_local_dir, '')
remote_path = path.replace(base_local_dir, base_remote_dir)
remote_path = remote_path.replace('\\', '/') # Convert to unix style
if not ftp_path_tools.ftp_exists(remote_path):
ftp_path_tools.ftp_mkdirs(remote_path)
# Change to directory
try:
ftp_h.cwd(remote_path)
continue_on = True
except ftplib.error_perm, e:
print 'ERROR -- %s' % (str(e.args))
except psftplib.PsFtpInvalidCommand, e:
print 'ERROR -- %s' % (str(e.args))
if continue_on:
if os.path.exists(filepath):
f_h = open(filepath,'rb')
filename = os.path.split(f_h.name)[-1]
display_filename = os.path.join(remote_sub_path, filename)
display_filename = display_filename.replace('\\', '/')
print 'Sending (%s) ...' % (display_filename),
send_cmd = 'STOR %s' % (filename)
try:
ftp_h.storbinary(send_cmd, f_h)
f_h.close()
print 'Done!'
except Exception, e:
print 'ERROR!'
print str(e.args)
print
else:
print "WARNING -- File no longer exists, (%s)!" % (filepath)
ftp_h.quit()
print 'Closing Connection'
else:
print 'ERROR -- No files found in (%s)' % (base_local_dir)
return continue_on
if __name__ == '__main__':
import optparse
default_config_file = u'ftpallcfg.py'
# Create parser, and configure command line options to parse
parser = optparse.OptionParser()
parser.add_option("-l", "--local_dir",
dest="local_dir",
help="Local Directory (Defaults to CWD)",
default='.')
parser.add_option("-r", "--remote_dir",
dest="remote_dir",
help="[REQUIRED] Target Remote directory",
default=None)
parser.add_option("-u", "--username",
dest="username",
help="[REQUIRED] username",
default=None)
parser.add_option("-s","--server",
dest="server",
help="[REQUIRED] Server Address",
default=None)
parser.add_option("-e", "--encrypt",
action="store_true",
dest="encrypt",
help="Use sftp",
default=False)
parser.add_option("-m",
action="store_true",
dest="monitor",
help="Keep process open and monitor changes",
default=False)
parser.add_option("-w",
action="store_true",
dest="walkdir",
help="Walk sub directories of the given directory to find files to send.",
default=False)
(options,args) = parser.parse_args()
if (options.username and options.server and options.remote_dir) or \
os.path.exists(default_config_file):
local_dir = options.local_dir
if os.path.exists(default_config_file):
sys.path.append('.')
import ftpallcfg
try:
server = ftpallcfg.server
username = ftpallcfg.username
remote_dir = ftpallcfg.remote_dir
encrypt = ftpallcfg.encrypt
monitor = ftpallcfg.monitor
walk = ftpallcfg.walk
except AttributeError, e:
print "ERROR --", str(e.args)
print
print 'Value(s) missing in %s file! The following values MUST be included:' % (default_config_file)
print '================================'
print 'server = <server to ftp to>'
print 'username = <Username for access to given server>'
print 'remote_dir = <remote server directory>'
print 'encrypt= True/False'
print 'monitor = True/False'
print 'walk == True/False'
print '================================'
sys.exit()
else:
server = options.server
username = options.username
remote_dir = options.remote_dir
encrypt = options.encrypt
monitor = options.monitor
walk = options.walkdir
# get the user password
prompt = 'Password (%s#%s): ' % (username, server)
if os.isatty(sys.stdin.fileno()):
p = getpass.getpass(prompt)
else:
#p = sys.stdin.readline().rstrip()
p = raw_input(prompt).rstrip()
if options.encrypt:
print '>> Using sftp for secure transfers <<'
print
if monitor:
try:
monitor_and_ftp(server,username,p,local_dir, remote_dir, encrypt, walk)
except KeyboardInterrupt:
print 'Exiting...'
else:
try:
upload_all(server, username, p, local_dir, remote_dir, [], encrypt, walk)
except KeyboardInterrupt:
print 'Exiting...'
else:
print 'ERROR -- Required option not given!'
print __revision__
print __doc__
print
parser.print_help()

EDIT 20/12/2017:
I have written a project in GitHub for this purpose. Click for details!
There are good answers above but i also want to add a good one using ftputil package. If you need to upload files from local directory to ftp directory, you can use this recursive function:
def upload_dir(localDir, ftpDir):
list = os.listdir(localDir)
for fname in list:
if os.path.isdir(localDir + fname):
if(ftp_host.path.exists(ftpDir + fname) != True):
ftp_host.mkdir(ftpDir + fname)
print(ftpDir + fname + " is created.")
upload_dir(localDir + fname + "/", ftpDir + fname + "/")
else:
if(ftp_host.upload_if_newer(localDir + fname, ftpDir + fname)):
print(ftpDir + fname + " is uploaded.")
else:
print(localDir + fname + " has already been uploaded.")
If you decide to use this function, you have to connect ftp using ftputil package. For this, you can use this snippet:
with ftputil.FTPHost("ftp_host", "ftp_username", "ftp_password") as ftp_host:
So, we're almost done. The last thing is usage of the function for beginners like me:
local_dir = "D:/Projects/.../"
ftp_dir = "/.../../"
upload_dir(local_dir, ftp_dir)
The most important thing is "/" character at the end of paths. You need to put it at the end. Finally, i want to share entire code:
with ftputil.FTPHost("ftp_host", "ftp_username", "ftp_password") as ftp_host:
def upload_dir(localDir, ftpDir):
list = os.listdir(localDir)
for fname in list:
if os.path.isdir(localDir + fname):
if(ftp_host.path.exists(ftpDir + fname) != True):
ftp_host.mkdir(ftpDir + fname)
print(ftpDir + fname + " is created.")
upload_dir(localDir + fname + "/", ftpDir + fname + "/")
else:
if(ftp_host.upload_if_newer(localDir + fname, ftpDir + fname)):
print(ftpDir + fname + " is uploaded.")
else:
print(localDir + fname + " has already been uploaded.")
local_dir = "D:/Projects/.../"
ftp_dir = "/.../../"
upload_dir(local_dir, ftp_dir)

Maybe you try ftpsync.py. If this one doesn't helps, try google search on python ftpsync and you get a lot of answers.

using ftputil:
import os
import ftputil
import ftputil.session
def upload_dir(root):
root = unicode(root, 'utf-8')
for dir_name, _, dir_files in os.walk(root):
local = os.path.join(os.curdir, dir_name)
remote = ftp_host.path.join(ftp_host.curdir, dir_name)
if not ftp_host.path.exists(remote):
print 'mkdir:', local, '->', remote
ftp_host.mkdir(remote)
for f in dir_files:
local_f = os.path.join(local, f)
remote_f = ftp_host.path.join(remote, f)
print 'upload:', local_f, '->', remote_f
ftp_host.upload(local_f, remote_f)
sf = ftputil.session.session_factory(use_passive_mode=True)
with ftputil.FTPHost('HOST', 'USER', 'PASS', session_factory=sf) as ftp_host:
upload_dir('DIR')

It is easy to use lftp to upload folders to an FTP. I use this in my Python script to move folders to FTP
Python script:
#! /usr/bin/python
import subprocess
subprocess.call(["bash", ftp_script, username, password, ftp, folder_to_move, src,folder_name_in_destination])
ftp_script:
lftp -u $1,$2 $3 <<EOF
mkdir $4
lcd $5
cd $6
mirror --reverse
EOF

Yet another answer using ftputil. And pathlib for figuring out the local path.
I've compiled and improved some of the above answers.
So thank you for the inspiration!
import os
import ftplib
import ftputil
from pathlib import Path
def ftp_upload(ftp_host, src_path, dst_path=None):
if dst_path is None:
ftp_host.path.join(os.curdir)
# Make sure that local path is a directory
if src_path.is_dir():
# Make sure that the root path exists
if ftp_host.path.exists(dst_path) is False:
# Inform user
print(f'mkdir: {src_path} -> {dst_path}')
# Create folder
ftp_host.mkdir(dst_path)
# Parse content of the root folder
for src_child in src_path.iterdir():
if src_child.is_dir():
# Use recursion for sub folders/files
dst_child = ftp_host.path.join(dst_path, src_child.stem)
ftp_upload(ftp_host, src_child, dst_child)
else:
# Copy sub files
dst_child = ftp_host.path.join(dst_path, src_child.name)
# Inform user
print(f'upload: {src_child} -> {dst_child}')
# Perform copy (upload)
ftp_host.upload(src_child, dst_child)
# ftp_host.upload_if_newer(src_child, dst_child)
else:
# Inform user
print(f"src_path: '{src_path}' must be an existing directory!")
if __name__ == '__main__':
# FTP settings
user = "ftp_user"
password = "ftp_password"
host = "localhost"
port = 2121
# Path settings
src_path = Path(__file__).parent.resolve() / "upload_dir"
dst_dir = "upload_dir"
# Inform user
print(f"Establishing FTP session as '{user}'")
# Establish FTP session
ftplib.FTP.port = port
with ftputil.FTPHost(host, user, password) as ftp_host:
# Perform recursive FTP upload
dst_path = ftp_host.path.join(os.curdir, dst_dir)
ftp_upload(ftp_host, src_path, dst_path)

Why wont my Simple CGI Server handle AJAX Post requests?

I have created my own CGI python server script (that serves on port 8000) by following a tutorial. The server works beautifully if I want to generate web pages from python scripts, or serve a native HTML page BUT it doesn't work for when I make an AJAX POST request?
If I make an AJAX request to the python file aaa.py (using the javascript below) my server prints out the following error text:
Code 501, message can only POST to to CGI scripts
"POST /aaa.py HTTP/1.1" 501 -
What do you think I need to do to allow my python cgi server to allow/handle AJAX requests?
My CGI server:
__version__ = "0.4"
__all__ = ["CGIHTTPRequestHandler"]
import os
import sys
import urllib
import BaseHTTPServer
import SimpleHTTPServer
import select
class CGIHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
"""Complete HTTP server with GET, HEAD and POST commands.
GET and HEAD also support running CGI scripts.
The POST command is *only* implemented for CGI scripts.
"""
# Determine platform specifics
have_fork = hasattr(os, 'fork')
have_popen2 = hasattr(os, 'popen2')
have_popen3 = hasattr(os, 'popen3')
# pretend we don't have these to force execution in process
have_fork = 0
# Make rfile unbuffered -- we need to read one line and then pass
# the rest to a subprocess, so we can't use buffered input.
rbufsize = 0
def do_POST(self):
"""Serve a POST request.
This is only implemented for CGI scripts.
"""
if self.is_cgi():
self.run_cgi()
else:
self.send_error(501, "Can only POST to CGI scripts")
def send_head(self):
"""Version of send_head that support CGI scripts"""
if self.is_cgi():
return self.run_cgi()
else:
return SimpleHTTPServer.SimpleHTTPRequestHandler.send_head(self)
def is_cgi(self):
"""Test whether self.path corresponds to a CGI script.
Return a tuple (dir, rest) if self.path requires running a
CGI script, None if not. Note that rest begins with a
slash if it is not empty.
The default implementation tests whether the path
begins with one of the strings in the list
self.cgi_directories (and the next character is a '/'
or the end of the string).
"""
path = self.path
for x in self.cgi_directories:
i = len(x)
if path[:i] == x and (not path[i:] or path[i] == '/'):
self.cgi_info = path[:i], path[i+1:]
return True
return False
cgi_directories = ['/cgi-bin', '/htbin']
def is_executable(self, path):
"""Test whether argument path is an executable file."""
return executable(path)
def is_python(self, path):
"""Test whether argument path is a Python script."""
head, tail = os.path.splitext(path)
return tail.lower() in (".py", ".pyw")
def run_cgi(self):
"""Execute a CGI script."""
dir, rest = self.cgi_info
i = rest.rfind('?')
if i >= 0:
rest, query = rest[:i], rest[i+1:]
else:
query = ''
# i = rest.find('/')
# if i >= 0:
# script, rest = rest[:i], rest[i:]
# else:
# script, rest = rest, ''
script = rest
scriptname = dir + '/' + script
scriptfile = self.translate_path(scriptname)
if not os.path.exists(scriptfile):
self.send_error(404, "No such CGI script (%s)" % `scriptname`)
return
if not os.path.isfile(scriptfile):
self.send_error(403, "CGI script is not a plain file (%s)" %
`scriptname`)
return
ispy = self.is_python(scriptname)
if not ispy:
if not (self.have_fork or self.have_popen2 or self.have_popen3):
self.send_error(403, "CGI script is not a Python script (%s)" %
`scriptname`)
return
if not self.is_executable(scriptfile):
self.send_error(403, "CGI script is not executable (%s)" %
`scriptname`)
return
# Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
# XXX Much of the following could be prepared ahead of time!
env = {}
env['SERVER_SOFTWARE'] = self.version_string()
env['SERVER_NAME'] = self.server.server_name
env['GATEWAY_INTERFACE'] = 'CGI/1.1'
env['SERVER_PROTOCOL'] = self.protocol_version
env['SERVER_PORT'] = str(self.server.server_port)
env['REQUEST_METHOD'] = self.command
uqrest = urllib.unquote(rest)
env['PATH_INFO'] = uqrest
env['PATH_TRANSLATED'] = self.translate_path(uqrest)
env['SCRIPT_NAME'] = scriptname
if query:
env['QUERY_STRING'] = query
host = self.address_string()
if host != self.client_address[0]:
env['REMOTE_HOST'] = host
env['REMOTE_ADDR'] = self.client_address[0]
# XXX AUTH_TYPE
# XXX REMOTE_USER
# XXX REMOTE_IDENT
if self.headers.typeheader is None:
env['CONTENT_TYPE'] = self.headers.type
else:
env['CONTENT_TYPE'] = self.headers.typeheader
length = self.headers.getheader('content-length')
if length:
env['CONTENT_LENGTH'] = length
accept = []
for line in self.headers.getallmatchingheaders('accept'):
if line[:1] in "\t\n\r ":
accept.append(line.strip())
else:
accept = accept + line[7:].split(',')
env['HTTP_ACCEPT'] = ','.join(accept)
ua = self.headers.getheader('user-agent')
if ua:
env['HTTP_USER_AGENT'] = ua
co = filter(None, self.headers.getheaders('cookie'))
if co:
env['HTTP_COOKIE'] = ', '.join(co)
# XXX Other HTTP_* headers
if not self.have_fork:
# Since we're setting the env in the parent, provide empty
# values to override previously set values
for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
'HTTP_USER_AGENT'):
###, 'HTTP_COOKIE' -- removed by S.
env.setdefault(k, "")
# for key in env.keys():
# print key + " '" + env[key] + "'"
os.environ.update(env)
self.send_response(200, "Script output follows")
decoded_query = query.replace('+', ' ')
if self.have_fork:
# Unix -- fork as we should
args = [script]
if '=' not in decoded_query:
args.append(decoded_query)
nobody = nobody_uid()
self.wfile.flush() # Always flush before forking
pid = os.fork()
if pid != 0:
# Parent
pid, sts = os.waitpid(pid, 0)
# throw away additional data [see bug #427345]
while select.select([self.rfile], [], [], 0)[0]:
if not self.rfile.read(1):
break
if sts:
self.log_error("CGI script exit status %#x", sts)
return
# Child
try:
try:
os.setuid(nobody)
except os.error:
pass
os.dup2(self.rfile.fileno(), 0)
os.dup2(self.wfile.fileno(), 1)
os.execve(scriptfile, args, os.environ)
except:
self.server.handle_error(self.request, self.client_address)
os._exit(127)
elif self.have_popen2 or self.have_popen3:
# Windows -- use popen2 or popen3 to create a subprocess
import shutil
if self.have_popen3:
popenx = os.popen3
else:
popenx = os.popen2
cmdline = scriptfile
if self.is_python(scriptfile):
interp = sys.executable
if interp.lower().endswith("w.exe"):
# On Windows, use python.exe, not pythonw.exe
interp = interp[:-5] + interp[-4:]
cmdline = "%s -u \"%s\"" % (interp, cmdline)
if '=' not in query and '"' not in query:
cmdline = '%s "%s"' % (cmdline, query)
self.log_message("command: %s", cmdline)
try:
nbytes = int(length)
except (TypeError, ValueError):
nbytes = 0
files = popenx(cmdline, 'b')
fi = files[0]
fo = files[1]
if self.have_popen3:
fe = files[2]
if self.command.lower() == "post" and nbytes > 0:
data = self.rfile.read(nbytes)
fi.write(data)
# throw away additional data [see bug #427345]
while select.select([self.rfile._sock], [], [], 0)[0]:
if not self.rfile._sock.recv(1):
break
fi.close()
shutil.copyfileobj(fo, self.wfile)
if self.have_popen3:
errors = fe.read()
fe.close()
if errors:
self.log_error('%s', errors)
sts = fo.close()
if sts:
self.log_error("CGI script exit status %#x", sts)
else:
self.log_message("CGI script exited OK")
else:
# Other O.S. -- execute script in this process
save_argv = sys.argv
save_stdin = sys.stdin
save_stdout = sys.stdout
save_stderr = sys.stderr
try:
try:
sys.argv = [scriptfile]
if '=' not in decoded_query:
sys.argv.append(decoded_query)
sys.stdout = self.wfile
sys.stdin = self.rfile
execfile(scriptfile, {"__name__": "__main__"})
finally:
sys.argv = save_argv
sys.stdin = save_stdin
sys.stdout = save_stdout
sys.stderr = save_stderr
except SystemExit, sts:
self.log_error("CGI script exit status %s", str(sts))
else:
self.log_message("CGI script exited OK")
nobody = None
def nobody_uid():
"""Internal routine to get nobody's uid"""
global nobody
if nobody:
return nobody
try:
import pwd
except ImportError:
return -1
try:
nobody = pwd.getpwnam('nobody')[2]
except KeyError:
nobody = 1 + max(map(lambda x: x[2], pwd.getpwall()))
return nobody
def executable(path):
"""Test for executable file."""
try:
st = os.stat(path)
except os.error:
return False
return st.st_mode & 0111 != 0
def test(HandlerClass = CGIHTTPRequestHandler,
ServerClass = BaseHTTPServer.HTTPServer):
SimpleHTTPServer.test(HandlerClass, ServerClass)
if __name__ == '__main__':
test()
Code in aaa.py:
#!/usr/bin/env python
import cgitb; cgitb.enable()
import cgi
import os
print "Content-Type: text/html\n"
input_data = cgi.FieldStorage()
print "hello"
My AJAX/ Javascript:
function onTest( dest, params )
{
var xmlhttp;
if (window.XMLHttpRequest)
{// code for IE7+, Firefox, Chrome, Opera, Safari
xmlhttp=new XMLHttpRequest();
}
else
{// code for IE6, IE5
xmlhttp=new ActiveXObject("Microsoft.XMLHTTP");
}
xmlhttp.onreadystatechange=function()
{
if (xmlhttp.readyState==4 && xmlhttp.status==200)
{
document.getElementById( "bb" ).innerHTML = xmlhttp.responseText;
}
}
xmlhttp.open("POST",dest,true);
xmlhttp.setRequestHeader("Content-type","application/x-www-form-urlencoded");
xmlhttp.send( params );
}

when you use the python http cgi server, cgi scripts have to be under the subdir (from you server script ) /cgi-bin