I'm trying to get a database backup from odoo erp using ftputil
this is the code
# -*- coding: utf-8 -*-
from odoo import models, fields, api, tools, _
from odoo.exceptions import Warning
import odoo
from odoo.http import content_disposition
import logging
_logger = logging.getLogger(__name__)
import os
import datetime
try:
from xmlrpc import client as xmlrpclib
except ImportError:
import xmlrpclib
import time
import base64
import socket
try:
import ftputil
except ImportError:
raise ImportError(
'This module needs ftputil to automatically write backups to the FTP through ftp. Please install ftputil on your system. (sudo pip3 install ftputil)')
def execute(connector, method, *args):
res = False
try:
res = getattr(connector, method)(*args)
except socket.error as error:
_logger.critical('Error while executing the method "execute". Error: ' + str(error))
raise error
return res
class db_backup(models.Model):
_name = 'db.backup'
#api.multi
def get_db_list(self, host, port, context={}):
uri = 'http://' + host + ':' + port
conn = xmlrpclib.ServerProxy(uri + '/xmlrpc/db')
db_list = execute(conn, 'list')
return db_list
#api.multi
def _get_db_name(self):
dbName = self._cr.dbname
return dbName
# Columns for local server configuration
host = fields.Char('Host', required=True, default='localhost')
port = fields.Char('Port', required=True, default=8069)
name = fields.Char('Database', required=True, help='Database you want to schedule backups for',
default=_get_db_name)
folder = fields.Char('Backup Directory', help='Absolute path for storing the backups', required='True',
default='/odoo/backups')
backup_type = fields.Selection([('zip', 'Zip'), ('dump', 'Dump')], 'Backup Type', required=True, default='zip')
autoremove = fields.Boolean('Auto. Remove Backups',
help='If you check this option you can choose to automaticly remove the backup after xx days')
days_to_keep = fields.Integer('Remove after x days',
help="Choose after how many days the backup should be deleted. For example:\nIf you fill in 5 the backups will be removed after 5 days.",
required=True)
# Columns for external server (SFTP)
sftp_write = fields.Boolean('Write to external server with sftp',
help="If you check this option you can specify the details needed to write to a remote server with SFTP.")
sftp_path = fields.Char('Path external server',
help='The location to the folder where the dumps should be written to. For example /odoo/backups/.\nFiles will then be written to /odoo/backups/ on your remote server.')
sftp_host = fields.Char('IP Address SFTP Server',
help='The IP address from your remote server. For example 192.168.0.1')
sftp_port = fields.Integer('SFTP Port', help='The port on the FTP server that accepts SSH/SFTP calls.', default=22)
sftp_user = fields.Char('Username SFTP Server',
help='The username where the SFTP connection should be made with. This is the user on the external server.')
sftp_password = fields.Char('Password User SFTP Server',
help='The password from the user where the SFTP connection should be made with. This is the password from the user on the external server.')
days_to_keep_sftp = fields.Integer('Remove SFTP after x days',
help='Choose after how many days the backup should be deleted from the FTP server. For example:\nIf you fill in 5 the backups will be removed after 5 days from the FTP server.',
default=30)
send_mail_sftp_fail = fields.Boolean('Auto. E-mail on backup fail',
help='If you check this option you can choose to automaticly get e-mailed when the backup to the external server failed.')
email_to_notify = fields.Char('E-mail to notify',
help='Fill in the e-mail where you want to be notified that the backup failed on the FTP.')
#api.multi
def _check_db_exist(self):
self.ensure_one()
db_list = self.get_db_list(self.host, self.port)
if self.name in db_list:
return True
return False
_constraints = [(_check_db_exist, _('Error ! No such database exists!'), [])]
#api.multi
def test_sftp_connection(self, context=None):
self.ensure_one()
# Check if there is a success or fail and write messages
messageTitle = ""
messageContent = ""
error = ""
has_failed = False
for rec in self:
db_list = self.get_db_list(rec.host, rec.port)
pathToWriteTo = rec.sftp_path
ipHost = rec.sftp_host
portHost = rec.sftp_port
usernameLogin = rec.sftp_user
passwordLogin = rec.sftp_password
# Connect with external server over SFTP, so we know sure that everything works.
try:
with ftputil.FTPHost(ipHost, usernameLogin, passwordLogin) as s:
messageTitle = _("Connection Test Succeeded!\nEverything seems properly set up for FTP back-ups!")
except Exception as e:
_logger.critical('There was a problem connecting to the remote ftp: ' + str(e))
error += str(e)
has_failed = True
messageTitle = _("Connection Test Failed!")
if len(rec.sftp_host) < 8:
messageContent += "\nYour IP address seems to be too short.\n"
messageContent += _("Here is what we got instead:\n")
finally:
if s:
s.close()
if has_failed:
raise Warning(messageTitle + '\n\n' + messageContent + "%s" % str(error))
else:
raise Warning(messageTitle + '\n\n' + messageContent)
#api.model
def schedule_backup(self):
conf_ids = self.search([])
for rec in conf_ids:
db_list = self.get_db_list(rec.host, rec.port)
if rec.name in db_list:
try:
if not os.path.isdir(rec.folder):
os.makedirs(rec.folder)
except:
raise
# Create name for dumpfile.
bkp_file = '%s_%s.%s' % (time.strftime('%Y_%m_%d_%H_%M_%S'), rec.name, rec.backup_type)
file_path = os.path.join(rec.folder, bkp_file)
uri = 'http://' + rec.host + ':' + rec.port
conn = xmlrpclib.ServerProxy(uri + '/xmlrpc/db')
bkp = ''
try:
# try to backup database and write it away
fp = open(file_path, 'wb')
odoo.service.db.dump_db(rec.name, fp, rec.backup_type)
fp.close()
except Exception as error:
_logger.debug(
"Couldn't backup database %s. Bad database administrator password for server running at http://%s:%s" % (
rec.name, rec.host, rec.port))
_logger.debug("Exact error from the exception: " + str(error))
continue
else:
_logger.debug("database %s doesn't exist on http://%s:%s" % (rec.name, rec.host, rec.port))
# Check if user wants to write to SFTP or not.
if rec.sftp_write is True:
try:
# Store all values in variables
dir = rec.folder
pathToWriteTo = rec.sftp_path
ipHost = rec.sftp_host
portHost = rec.sftp_port
usernameLogin = rec.sftp_user
passwordLogin = rec.sftp_password
_logger.debug('sftp remote path: %s' % pathToWriteTo)
try:
with ftputil.FTPHost(ipHost, usernameLogin, passwordLogin) as sftp:
pass
except Exception as error:
_logger.critical('Error connecting to remote server! Error: ' + str(error))
try:
sftp.chdir(pathToWriteTo)
except IOError:
# Create directory and subdirs if they do not exist.
currentDir = ''
for dirElement in pathToWriteTo.split('/'):
currentDir += dirElement + '/'
try:
sftp.chdir(currentDir)
except:
_logger.info('(Part of the) path didn\'t exist. Creating it now at ' + currentDir)
# Make directory and then navigate into it
sftp.mkdir(currentDir, 777)
sftp.chdir(currentDir)
pass
sftp.chdir(pathToWriteTo)
# Loop over all files in the directory.
for f in os.listdir(dir):
if rec.name in f:
fullpath = os.path.join(dir, f)
if os.path.isfile(fullpath):
try:
sftp.StatResult(os.path.join(pathToWriteTo, f))
_logger.debug(
'File %s already exists on the remote FTP Server ------ skipped' % fullpath)
# This means the file does not exist (remote) yet!
except IOError:
try:
# sftp.put(fullpath, pathToWriteTo)
sftp.upload(fullpath, os.path.join(pathToWriteTo, f))
_logger.info('Copying File % s------ success' % fullpath)
except Exception as err:
_logger.critical(
'We couldn\'t write the file to the remote server. Error: ' + str(err))
# Navigate in to the correct folder.
sftp.chdir(pathToWriteTo)
# Loop over all files in the directory from the back-ups.
# We will check the creation date of every back-up.
for file in sftp.listdir(pathToWriteTo):
if rec.name in file:
# Get the full path
fullpath = os.path.join(pathToWriteTo, file)
# Get the timestamp from the file on the external server
timestamp = sftp.StatResult(fullpath).st_atime
createtime = datetime.datetime.fromtimestamp(timestamp)
now = datetime.datetime.now()
delta = now - createtime
# If the file is older than the days_to_keep_sftp (the days to keep that the user filled in on the Odoo form it will be removed.
if delta.days >= rec.days_to_keep_sftp:
# Only delete files, no directories!
if sftp.isfile(fullpath) and (".dump" in file or '.zip' in file):
_logger.info("Delete too old file from SFTP servers: " + file)
sftp.unlink(file)
# Close the SFTP session.
sftp.close()
except Exception as e:
_logger.debug('Exception! We couldn\'t back up to the FTP server..')
# At this point the SFTP backup failed. We will now check if the user wants
# an e-mail notification about this.
if rec.send_mail_sftp_fail:
try:
ir_mail_server = self.env['ir.mail_server']
message = "Dear,\n\nThe backup for the server " + rec.host + " (IP: " + rec.sftp_host + ") failed.Please check the following details:\n\nIP address SFTP server: " + rec.sftp_host + "\nUsername: " + rec.sftp_user + "\nPassword: " + rec.sftp_password + "\n\nError details: " + tools.ustr(
e) + "\n\nWith kind regards"
msg = ir_mail_server.build_email("auto_backup#" + rec.name + ".com", [rec.email_to_notify],
"Backup from " + rec.host + "(" + rec.sftp_host + ") failed",
message)
ir_mail_server.send_email(self._cr, self._uid, msg)
except Exception:
pass
"""
Remove all old files (on local server) in case this is configured..
"""
if rec.autoremove:
dir = rec.folder
# Loop over all files in the directory.
for f in os.listdir(dir):
fullpath = os.path.join(dir, f)
# Only delete the ones which are from the current database
# (Makes it possible to save different databases in the same folder)
if rec.name in fullpath:
timestamp = os.stat(fullpath).st_ctime
createtime = datetime.datetime.fromtimestamp(timestamp)
now = datetime.datetime.now()
delta = now - createtime
if delta.days >= rec.days_to_keep:
# Only delete files (which are .dump and .zip), no directories.
if os.path.isfile(fullpath) and (".dump" in f or '.zip' in f):
_logger.info("Delete local out-of-date file: " + fullpath)
os.remove(fullpath)
I cant get pass this logger "_logger.critical(
'We couldn't write the file to the remote server. Error: ' + str(err))"
So my project is that I need to send a jpg image from one computer to another computer in the same network. To send the data I split the data into chunks of at least 9999 bytes and then I create a length header that tells the length of the data and I attach it to the start of the massage. here is the code:
the protocol:
import os.path
LENGTH_FIELD_SIZE = 4
PORT = 8820
COMANDS_LIST = "TAKE_SCREENSHOT\nSEND_PHOTO\nDIR\nDELETE\nCOPY\nEXECUTE\nEXIT".split("\n")
def check_cmd(data):
"""
Check if the command is defined in the protocol, including all parameters
For example, DELETE c:\work\file.txt is good, but DELETE alone is not
"""
command = ""
file_location =""
splited_data = data.split(maxsplit=1)
if len(splited_data) == 2:
command, file_location = splited_data
return (command in COMANDS_LIST) and (file_location is not None)
elif len(splited_data) == 1:
command = splited_data[0]
return command in ["TAKE_SCREENSHOT","EXIT","SEND_PHOTO"]
return False
# (3)
def create_msg(data):
"""
Create a valid protocol message, with length field
"""
data_len = len(str(data))
if data_len > 9999 or data_len == 0:
print(f"data len is bigger then 9999 or is 0, data len = {data_len} ")
return False
len_field = str(data_len).zfill(4)
# (4)
print(len_field)
return True ,f"{len_field}{data}"
def get_msg(my_socket):
"""
Extract message from protocol, without the length field
If length field does not include a number, returns False, "Error"
"""
lenght_field = ""
data = ""
try:
while len(lenght_field) < 4:
lenght_field += my_socket.recv(4).decode()
except RuntimeError as exc_run:
return False, "header wasnt sent properly"
if not lenght_field.isdigit():
return False, "error, length header is not valid"
lenght_field = lenght_field.lstrip("0")
while len(data) < int(lenght_field):
data += my_socket.recv(int(lenght_field)).decode()
return True, data
now the protocol works fine when I use the same computer for both server and client and when I debug get_msg on the other computer. when I'm not, it seems that the problem is that the part that recv the header will recv something else after a few successful recv and return an error message.
here are the server parts:
import socket
import pyautogui as pyautogui
import protocol
import glob
import os.path
import shutil
import subprocess
import base64
IP = "0.0.0.0"
PORT = 8820
PHOTO_PATH = r"C:\Users\Innon\Pictures\Screenshots\screenShot.jpg"# The path + filename where the screenshot at the server should be saved
def check_client_request(cmd):
"""
Break cmd to command and parameters
Check if the command and params are good.
For example, the filename to be copied actually exists
Returns:
valid: True/False
command: The requested cmd (ex. "DIR")
params: List of the cmd params (ex. ["c:\\cyber"])
"""
# Use protocol.check_cmd first
cmd_arr = cmd.split(maxsplit=1)
command = cmd_arr[0]
file_location = None
if len(cmd_arr) == 2:
file_location = cmd_arr[1]
if file_location == None:
return protocol.check_cmd(cmd) ,command, file_location
else:
file_location = tuple(str(file_location).split())
if (os.path.exists(file_location[0])):
return protocol.check_cmd(cmd) , command , file_location
return False , command , file_location
# Then make sure the params are valid
# (6)
def handle_client_request(command,params):
"""Create the response to the client, given the command is legal and params are OK
For example, return the list of filenames in a directory
Note: in case of SEND_PHOTO, only the length of the file will be sent
Returns:
response: the requested data
"""
# (7)
response = "no server response"
if command == "DIR":
response = glob.glob(f"{params[0]}\\*.*" )
if command == "DELETE":
os.remove(params[0])
response = f"{params[0]} was deleted"
if command == "COPY":
try:
shutil.copy(params[0],params[1])
response = f"{params[0]} was copyed to {params[1]}"
except FileNotFoundError as ex1:
response = ex1
except IndexError as ex2:
response = ex2
if command == "EXECUTE":
subprocess.call(params[0])
response = f"{params[0]} was executed"
if command == "TAKE_SCREENSHOT":
#todo find a way to know and create the locatipn of screen shot to be saved
myScreenshot = pyautogui.screenshot()
myScreenshot.save(PHOTO_PATH)
response = f"screen shot have been taken and been saved at {PHOTO_PATH}"
if command == "SEND_PHOTO":
with open(PHOTO_PATH, "rb") as file:
file_data = base64.b64encode(file.read()).decode()
print(file_data)
is_vaild_response, img_length = protocol.create_msg(len(file_data))
print(img_length)
img_data = ""
if not is_vaild_response:
response = "img length data isnt valid"
return response
while len(file_data) > 0:
chunk_data = file_data[:9999]
is_vaild_response, data = protocol.create_msg(chunk_data)
if not is_vaild_response:
response = "img data isnt valid"
return response
img_data += data
file_data = file_data[9999:]
response = f"{img_length}{img_data}"
return response
def main():
# open socket with client
server_socket = socket.socket()
server_socket.bind((IP,PORT))
server_socket.listen(1)
# (1)
client_socket, addr = server_socket.accept()
# handle requests until user asks to exit
while True:
# Check if protocol is OK, e.g. length field OK
valid_protocol, cmd = protocol.get_msg(client_socket)
print(f"got message {valid_protocol}")
if valid_protocol:
# Check if params are good, e.g. correct number of params, file name exists
valid_cmd, command, params = check_client_request(cmd)
print(f"check_client_request {valid_cmd}")
if valid_cmd:
# (6)
if command == 'EXIT':
break
if command == 'SEND_PHOTO':
data = handle_client_request(command, params)
client_socket.sendall(data.encode())
continue
# prepare a response using "handle_client_request"
data = handle_client_request(command,params)
# add length field using "create_msg"
is_vaild_response , response = protocol.create_msg(data)
print(f"creat_msg {is_vaild_response}")
# send to client
if is_vaild_response:
client_socket.sendall(response.encode())
else:
# prepare proper error to client
resp = 'Bad command or parameters'
is_vaild_response , response = protocol.create_msg(resp)
# send to client
client_socket.sendall(response.encode())
else:
# prepare proper error to client
resp = 'Packet not according to protocol'
is_vaild_response, response = protocol.create_msg(resp)
#send to client
client_socket.sendall(response.encode())
# Attempt to clean garbage from socket
client_socket.recv(1024)
# close sockets
resp = "Closing connection"
print(resp)
is_vaild_response, response = protocol.create_msg(resp)
client_socket.sendall(response.encode())
client_socket.close()
server_socket.close()
if __name__ == '__main__':
main()
and the client:
import socket
import base64
import protocol
IP = "127.0.0.1"
SAVED_PHOTO_LOCATION = r'C:\Users\Innon\Pictures\Saved Pictures\screenShot.jpg' # The path + filename where the copy of the screenshot at the client should be saved
def handle_server_response(my_socket, cmd):
"""
Receive the response from the server and handle it, according to the request
For example, DIR should result in printing the contents to the screen,
Note- special attention should be given to SEND_PHOTO as it requires and extra receive
"""
# (8) treat all responses except SEND_PHOTO
if "SEND_PHOTO" not in cmd:
vaild_data, data = protocol.get_msg(my_socket)
if vaild_data:
return data
# (10) treat SEND_PHOTO
else:
pic_data = ""
vaild_pick_len, pic_len = protocol.get_msg(my_socket)
if pic_len.isdigit() == False:
print(f"picture length is not valid. got massage: {pic_len}")
return
with open(SAVED_PHOTO_LOCATION, "wb") as file:
while len(pic_data) < int(pic_len):
vaild_data, data = protocol.get_msg(my_socket)
if not vaild_data:
return f"img data isnt valid. {data}"
pic_data += data
print(pic_data)
file.write(base64.b64decode(pic_data.encode()))
return "img was recived succesfully "
def main():
# open socket with the server
my_socket = socket.socket()
my_socket.connect((IP,8820))
# (2)
# print instructions
print('Welcome to remote computer application. Available commands are:\n')
print('TAKE_SCREENSHOT\nSEND_PHOTO\nDIR\nDELETE\nCOPY\nEXECUTE\nEXIT')
# loop until user requested to exit
while True:
cmd = input("Please enter command:\n")
if protocol.check_cmd(cmd):
valid_pack , packet = protocol.create_msg(cmd)
if valid_pack:
my_socket.sendall(packet.encode())
print(handle_server_response(my_socket, cmd))
if cmd == 'EXIT':
break
else:
print("Not a valid command, or missing parameters\n")
my_socket.close()
if __name__ == '__main__':
main()
here is how the problem looks like:thi is how it looks
here is how to needs look like:
the right way
thank you.
the solution was to change get_msg function in the protocol:
while len(data) < int(lenght_field):
data += my_socket.recv(int(lenght_field) - len(data)).decode()
instead of:
while len(data) < int(lenght_field):
data += my_socket.recv(int(lenght_field)).decode()
I'm trying to take an image via webcam then send the image through gmail. That function suppose to happen if my fingerprint is known at the system. If it does it needs to execute the send email function. I added after the send email function to send 'HIGH' to a gpio. I don't think it suppose to interfere.
When there is a known template it is followed by an "else:" statement. then I put my "def find_newest_file(dir):" function. What am I doing wrong?
might note I'm terrible at programming. Thank you!
Pyfingerprint
import os
import glob
import time
import smtplib
import imghdr
from email.message import EmailMessage
import RPi.GPIO as GPIO
from time import sleep
GPIO.setmode(GPIO.BCM)
GPIO.setup(26, GPIO.OUT)
import hashlib
from pyfingerprint.pyfingerprint import PyFingerprint
GPIO.output(26, 0)
## Search for a finger
##
## Tries to initialize the sensor
try:
f = PyFingerprint('/dev/ttyUSB0', 57600, 0xFFFFFFFF, 0x00000000)
if ( f.verifyPassword() == False ):
raise ValueError('The given fingerprint sensor password is wrong!')
except Exception as e:
print('The fingerprint sensor could not be initialized!')
print('Exception message: ' + str(e))
exit(1)
## Gets some sensor information
print('Currently used templates: ' + str(f.getTemplateCount()) +'/'+ str(f.getStorageCapacity()))
## Tries to search the finger and calculate hash
try:
print('Waiting for finger...')
## Wait that finger is read
while ( f.readImage() == False ):
pass
## Converts read image to characteristics and stores it in charbuffer 1
f.convertImage(0x01)
## Searchs template
result = f.searchTemplate()
positionNumber = result[0]
accuracyScore = result[1]
if ( positionNumber == -1 ):
print('No match found!')
exit(0)
else:
print('Found template at position #' + str(positionNumber))
print('The accuracy score is: ' + str(accuracyScore))
def find_newest_file(dir):
os.system('fswebcam -r 1280x720 -S 3 --jpeg 90 --save /home/pi/Pictures/%H%M%S.jpg')
types = ['*.png', '*.jpg']
files = []
if not os.path.isdir(dir):
print(f'ERROR: {dir} does not exist. or it is not a valid folder')
exit()
for ext in types:
scan_path = os.path.abspath(os.path.join(dir, ext))
files.extend(glob.glob(scan_path))
if len(files) == 0:
print(f'ERROR: file not found while scanning folder: {dir} for: {types}')
exit()
newest = None
n_time = 0
for file in files:
# print(file)
c_time = os.path.getctime(file)
if c_time > n_time:
n_time = c_time
newest = file
if newest is None:
print(f'-----------\nUnexpected error: None was return while the list was not empty:\n{files}')
exit()
if os.path.exists(newest):
return newest # return as a list since this is what Yehonata expect
else:
print(f'ERROR: File {newest} not found')
exit()
# Yehontan Code
Sender_Email = "Sender#gmail.com"
Reciever_Email = "reciver#gmailcom"
Password = input('Enter your email account password: ')
newMessage = EmailMessage()
newMessage['Subject'] = "Check out the new logo"
newMessage['From'] = Sender_Email
newMessage['To'] = Reciever_Email
newMessage.set_content('Let me know what you think. Image attached!')
# Tomerl: Replace static name with seach function
files = [ find_newest_file('/home/pi/Pictures/') ]
for file in files:
with open(file, 'rb') as f:
image_data = f.read()
image_type = imghdr.what(f.name)
image_name = f.name
newMessage.add_attachment(image_data, maintype='image', subtype=image_type, filename=image_name)
with smtplib.SMTP_SSL('smtp.gmail.com', 465) as smtp:
smtp.login(Sender_Email, Password)
smtp.send_message(newMessage)
try:
for x in range(1):
GPIO.output(26, 1)
sleep(2.5)
GPIO.output(26, 0)
sleep(2.5)
except KeyboardInterrupt:
GPIO.cleanup()
GPIO.output(26, 0)
## Loads the found template to charbuffer 1
f.loadTemplate(positionNumber, 0x01)
## Downloads the characteristics of template loaded in charbuffer 1
characterics = str(f.downloadCharacteristics(0x01)).encode('utf-8')
## Hashes characteristics of template
print('SHA-2 hash of template: ' + hashlib.sha256(characterics).hexdigest())
except Exception as e:
print('Operation failed!')
print('Exception message: ' + str(e))
exit(1)
You need to define the function at the top of the program. You only need to call it after the else statement. Basically, put
def find_newest_file(dir):
os.system('fswebcam -r 1280x720 -S 3 --jpeg 90 --save /home/pi/Pictures/%H%M%S.jpg')
types = ['*.png', '*.jpg']
files = []
if not os.path.isdir(dir):
print(f'ERROR: {dir} does not exist. or it is not a valid folder')
exit()
at the top and when you want to use it do
find_newest_file(foo)
Good luck on your journey!
I'm a networking student who has little programming experience and I have simple project on my hands where I need to make a simple notification that notifies the user when an email arrives in gmail. I have been trying to get this to work for the last few weeks and I am honestly stuck and struggling. I'm using the code below and I think the problem is the authentication bit, where I can't connect to the server. All help is appreciated.
import imaplib
import re
import time
import subprocess
## enter your account details bellow!
imapServer = "imap.gmail.com"
port = "993"
username = "example#gmail.com"
password = "password"
##how often to check ? give interval in seconds! Checking too often might performance and stability.
checkInterval = 120
Mailbox = imaplib.IMAP4_SSL(imapServer, port)
rc,resp = Mailbox.login(username,password)
if rc == 'OK':
print("Connected to mail-server " + imapServer)
rc, message = Mailbox.status('INBOX', "(UNSEEN)")
unreadCount = int(re.search("UNSEEN (\d+)",str( message[0])).group(1))
oldValue = 0
file = open("%sytemdrive%\Windows\Temp\mailnotify.tmp", "w+")
file.write(str(unreadCount))
file.close
while(1):
rc, message = Mailbox.status('INBOX', "(UNSEEN)")
unreadCount = int(re.search("UNSEEN (\d+)",str( message[0])).group(1))
file = open("%sytemdrive%\Windows\Temp\mailnotify.tmp", "w+")
oldValue = int(file.readline())
file.close()
if (unreadCount>oldValue):
subprocess.call(["notify-send", "-u", "low", "low", "t", "5000", "New email!", "New email!",
"You have " + str(unreadCount) + " unread " + "emails!" if unreadCount > 1 else "email!",
"--icon=email"])
if oldValue != unreadCount:
file = open("%sytemdrive%\Windows\Temp\mailnotify.tmp", "w+")
file.write(str(unreadCount))
file.close()
time.sleep(checkInterval)
else :
print('Fail to connect')
Mailbox.logout()
file.remove()
I have to automatically upload folders to an FTP using a Python script. I am able to upload a single file, but not folders with subfolders and files in them. I did a lot of search, but failed. Could some one help me out here? Thanks in advance.
#! /usr/bin/python
import ftplib
s = ftplib.FTP('serverip','usrname','password')
file = '/home/rock/test.txt'
ftppath = '/IT'
filename = "rak"
s.cwd(ftppath)
f = open(file,'rb')
s.storbinary('STOR ' + filename, f)
f.close()
s.quit()
I recently came into this problem and figured out a recursive function to solve it.
import ftplib
import os
server = 'localhost'
username = 'generic_user'
password = 'password'
myFTP = ftplib.FTP(server, username, password)
myPath = r'c:\temp'
def uploadThis(path):
files = os.listdir(path)
os.chdir(path)
for f in files:
if os.path.isfile(path + r'\{}'.format(f)):
fh = open(f, 'rb')
myFTP.storbinary('STOR %s' % f, fh)
fh.close()
elif os.path.isdir(path + r'\{}'.format(f)):
myFTP.mkd(f)
myFTP.cwd(f)
uploadThis(path + r'\{}'.format(f))
myFTP.cwd('..')
os.chdir('..')
uploadThis(myPath) # now call the recursive function
You basically need to use os.walk() to grab those files and transfer them.
Here's a script I wrote for myself to do much of what your asking.
I wrote it a long time ago, so I'd probably do it differently if I wrote it again, but I get a lot of use out of it.
It imports psftplib, which is a wrapper I wrote for the putty sftp.
Feel free to remove these references, or grab the lib at:
http://code.google.com/p/psftplib/source/browse/trunk/psftplib.py
# -*- coding: utf8 -*-
'''This tool will ftp all the files in a given directory to a given location
if the file ftpallcfg.py exists in the directory it will be loaded and the values within it used,
with the current directory used as the source directory.
ftpallcfg.py file contains the following variables.
===========================
server = <server to ftp to>
username = <Username for access to given server>
remote_dir = <remote server directory>
encrypt= True/False
monitor = True/False
walk = True/False
===========================
'''
import ftplib
import os
import getpass
import sys
import time
import socket
import psftplib
__revision__ = 1.11
SLEEP_SECONDS = 1
class FtpAddOns():
PATH_CACHE = []
def __init__(self, ftp_h):
self.ftp_h = ftp_h
def ftp_exists(self, path):
'''path exists check function for ftp handler'''
exists = None
if path not in self.PATH_CACHE:
try:
self.ftp_h.cwd(path)
exists = True
self.PATH_CACHE.append(path)
except ftplib.error_perm, e:
if str(e.args).count('550'):
exists = False
else:
exists = True
return exists
def ftp_mkdirs(self, path, sep='/'):
'''mkdirs function for ftp handler'''
split_path = path.split(sep)
new_dir = ''
for server_dir in split_path:
if server_dir:
new_dir += sep + server_dir
if not self.ftp_exists(new_dir):
try:
print 'Attempting to create directory (%s) ...' % (new_dir),
self.ftp_h.mkd(new_dir)
print 'Done!'
except Exception, e:
print 'ERROR -- %s' % (str(e.args))
def _get_local_files(local_dir, walk=False):
'''Retrieve local files list
result_list == a list of dictionaries with path and mtime keys. ex: {'path':<filepath>,'mtime':<file last modified time>}
ignore_dirs == a list of directories to ignore, should not include the base_dir.
ignore_files == a list of files to ignore.
ignore_file_ext == a list of extentions to ignore.
'''
result_list = []
ignore_dirs = ['CVS', '.svn']
ignore_files = ['.project', '.pydevproject']
ignore_file_ext = ['.pyc']
base_dir = os.path.abspath(local_dir)
for current_dir, dirs, files in os.walk(base_dir):
for this_dir in ignore_dirs:
if this_dir in dirs:
dirs.remove(this_dir)
sub_dir = current_dir.replace(base_dir, '')
if not walk and sub_dir:
break
for this_file in files:
if this_file not in ignore_files and os.path.splitext(this_file)[-1].lower() not in ignore_file_ext:
filepath = os.path.join(current_dir, this_file)
file_monitor_dict = {
'path': filepath,
'mtime': os.path.getmtime(filepath)
}
result_list.append(file_monitor_dict)
return result_list
def monitor_and_ftp(server,
username,
password,
local_dir,
remote_dir,
encrypt=False,
walk=False):
'''Monitor local files and when an update is found connect and upload'''
print 'Monitoring changes in (%s).' % (os.path.abspath(local_dir))
print '(Use ctrl-c to exit)'
last_files_list = _get_local_files(local_dir)
while True:
try:
time.sleep(SLEEP_SECONDS)
latest_files_list = _get_local_files(local_dir)
files_to_update = []
for idx in xrange(len(latest_files_list)):
if idx < len(last_files_list):
# compare last modified times
if latest_files_list[idx]['mtime'] > last_files_list[idx]['mtime']:
files_to_update.append(latest_files_list[idx])
else:
# add the file to the list (new file)
files_to_update.append(latest_files_list[idx])
if files_to_update:
print
print 'Detected NEW or CHANGED file(s), attempting to send ...'
print
is_success = upload_all(server,
username,
password,
local_dir,
remote_dir,
files_to_update,
encrypt,
walk)
if not is_success:
break
else:
print '.',
last_files_list = latest_files_list[:] # copy the list to hold
except KeyboardInterrupt:
print
print 'Exiting.'
break
def upload_all(server,
username,
password,
base_local_dir,
base_remote_dir,
files_to_update=None,
encrypt=False,
walk=False):
'''Upload all files in a given directory to the given remote directory'''
continue_on = False
login_ok = False
server_connect_ok = False
base_local_dir = os.path.abspath(base_local_dir)
base_remote_dir = os.path.normpath(base_remote_dir)
if files_to_update:
local_files = files_to_update
else:
local_files = _get_local_files(base_local_dir, walk)
if local_files:
if not encrypt: # Use standard FTP
ftp_h = ftplib.FTP()
else: # Use sftp
ftp_h = psftplib.SFTP()
try:
ftp_h.connect(server)
server_connect_ok = True
except socket.gaierror, e:
print 'ERROR -- Could not connect to (%s): %s' % (server, str(e.args))
except IOError, e:
print 'ERROR -- File not found: %s' % (str(e.args))
except socket.error, e:
print 'ERROR -- Could not connect to (%s): %s' % (server, str(e.args))
ftp_path_tools = FtpAddOns(ftp_h)
if server_connect_ok:
try:
ftp_h.login(username,password)
print 'Logged into (%s) as (%s)' % (server, username)
login_ok = True
except ftplib.error_perm, e:
print 'ERROR -- Check Username/Password: %s' % (str(e.args))
except psftplib.ProcessTimeout, e:
print 'ERROR -- Check Username/Password (timeout): %s' % (str(e.args))
if login_ok:
for file_info in local_files:
filepath = file_info['path']
path, filename = os.path.split(filepath)
remote_sub_path = path.replace(base_local_dir, '')
remote_path = path.replace(base_local_dir, base_remote_dir)
remote_path = remote_path.replace('\\', '/') # Convert to unix style
if not ftp_path_tools.ftp_exists(remote_path):
ftp_path_tools.ftp_mkdirs(remote_path)
# Change to directory
try:
ftp_h.cwd(remote_path)
continue_on = True
except ftplib.error_perm, e:
print 'ERROR -- %s' % (str(e.args))
except psftplib.PsFtpInvalidCommand, e:
print 'ERROR -- %s' % (str(e.args))
if continue_on:
if os.path.exists(filepath):
f_h = open(filepath,'rb')
filename = os.path.split(f_h.name)[-1]
display_filename = os.path.join(remote_sub_path, filename)
display_filename = display_filename.replace('\\', '/')
print 'Sending (%s) ...' % (display_filename),
send_cmd = 'STOR %s' % (filename)
try:
ftp_h.storbinary(send_cmd, f_h)
f_h.close()
print 'Done!'
except Exception, e:
print 'ERROR!'
print str(e.args)
print
else:
print "WARNING -- File no longer exists, (%s)!" % (filepath)
ftp_h.quit()
print 'Closing Connection'
else:
print 'ERROR -- No files found in (%s)' % (base_local_dir)
return continue_on
if __name__ == '__main__':
import optparse
default_config_file = u'ftpallcfg.py'
# Create parser, and configure command line options to parse
parser = optparse.OptionParser()
parser.add_option("-l", "--local_dir",
dest="local_dir",
help="Local Directory (Defaults to CWD)",
default='.')
parser.add_option("-r", "--remote_dir",
dest="remote_dir",
help="[REQUIRED] Target Remote directory",
default=None)
parser.add_option("-u", "--username",
dest="username",
help="[REQUIRED] username",
default=None)
parser.add_option("-s","--server",
dest="server",
help="[REQUIRED] Server Address",
default=None)
parser.add_option("-e", "--encrypt",
action="store_true",
dest="encrypt",
help="Use sftp",
default=False)
parser.add_option("-m",
action="store_true",
dest="monitor",
help="Keep process open and monitor changes",
default=False)
parser.add_option("-w",
action="store_true",
dest="walkdir",
help="Walk sub directories of the given directory to find files to send.",
default=False)
(options,args) = parser.parse_args()
if (options.username and options.server and options.remote_dir) or \
os.path.exists(default_config_file):
local_dir = options.local_dir
if os.path.exists(default_config_file):
sys.path.append('.')
import ftpallcfg
try:
server = ftpallcfg.server
username = ftpallcfg.username
remote_dir = ftpallcfg.remote_dir
encrypt = ftpallcfg.encrypt
monitor = ftpallcfg.monitor
walk = ftpallcfg.walk
except AttributeError, e:
print "ERROR --", str(e.args)
print
print 'Value(s) missing in %s file! The following values MUST be included:' % (default_config_file)
print '================================'
print 'server = <server to ftp to>'
print 'username = <Username for access to given server>'
print 'remote_dir = <remote server directory>'
print 'encrypt= True/False'
print 'monitor = True/False'
print 'walk == True/False'
print '================================'
sys.exit()
else:
server = options.server
username = options.username
remote_dir = options.remote_dir
encrypt = options.encrypt
monitor = options.monitor
walk = options.walkdir
# get the user password
prompt = 'Password (%s#%s): ' % (username, server)
if os.isatty(sys.stdin.fileno()):
p = getpass.getpass(prompt)
else:
#p = sys.stdin.readline().rstrip()
p = raw_input(prompt).rstrip()
if options.encrypt:
print '>> Using sftp for secure transfers <<'
print
if monitor:
try:
monitor_and_ftp(server,username,p,local_dir, remote_dir, encrypt, walk)
except KeyboardInterrupt:
print 'Exiting...'
else:
try:
upload_all(server, username, p, local_dir, remote_dir, [], encrypt, walk)
except KeyboardInterrupt:
print 'Exiting...'
else:
print 'ERROR -- Required option not given!'
print __revision__
print __doc__
print
parser.print_help()
EDIT 20/12/2017:
I have written a project in GitHub for this purpose. Click for details!
There are good answers above but i also want to add a good one using ftputil package. If you need to upload files from local directory to ftp directory, you can use this recursive function:
def upload_dir(localDir, ftpDir):
list = os.listdir(localDir)
for fname in list:
if os.path.isdir(localDir + fname):
if(ftp_host.path.exists(ftpDir + fname) != True):
ftp_host.mkdir(ftpDir + fname)
print(ftpDir + fname + " is created.")
upload_dir(localDir + fname + "/", ftpDir + fname + "/")
else:
if(ftp_host.upload_if_newer(localDir + fname, ftpDir + fname)):
print(ftpDir + fname + " is uploaded.")
else:
print(localDir + fname + " has already been uploaded.")
If you decide to use this function, you have to connect ftp using ftputil package. For this, you can use this snippet:
with ftputil.FTPHost("ftp_host", "ftp_username", "ftp_password") as ftp_host:
So, we're almost done. The last thing is usage of the function for beginners like me:
local_dir = "D:/Projects/.../"
ftp_dir = "/.../../"
upload_dir(local_dir, ftp_dir)
The most important thing is "/" character at the end of paths. You need to put it at the end. Finally, i want to share entire code:
with ftputil.FTPHost("ftp_host", "ftp_username", "ftp_password") as ftp_host:
def upload_dir(localDir, ftpDir):
list = os.listdir(localDir)
for fname in list:
if os.path.isdir(localDir + fname):
if(ftp_host.path.exists(ftpDir + fname) != True):
ftp_host.mkdir(ftpDir + fname)
print(ftpDir + fname + " is created.")
upload_dir(localDir + fname + "/", ftpDir + fname + "/")
else:
if(ftp_host.upload_if_newer(localDir + fname, ftpDir + fname)):
print(ftpDir + fname + " is uploaded.")
else:
print(localDir + fname + " has already been uploaded.")
local_dir = "D:/Projects/.../"
ftp_dir = "/.../../"
upload_dir(local_dir, ftp_dir)
Maybe you try ftpsync.py. If this one doesn't helps, try google search on python ftpsync and you get a lot of answers.
using ftputil:
import os
import ftputil
import ftputil.session
def upload_dir(root):
root = unicode(root, 'utf-8')
for dir_name, _, dir_files in os.walk(root):
local = os.path.join(os.curdir, dir_name)
remote = ftp_host.path.join(ftp_host.curdir, dir_name)
if not ftp_host.path.exists(remote):
print 'mkdir:', local, '->', remote
ftp_host.mkdir(remote)
for f in dir_files:
local_f = os.path.join(local, f)
remote_f = ftp_host.path.join(remote, f)
print 'upload:', local_f, '->', remote_f
ftp_host.upload(local_f, remote_f)
sf = ftputil.session.session_factory(use_passive_mode=True)
with ftputil.FTPHost('HOST', 'USER', 'PASS', session_factory=sf) as ftp_host:
upload_dir('DIR')
It is easy to use lftp to upload folders to an FTP. I use this in my Python script to move folders to FTP
Python script:
#! /usr/bin/python
import subprocess
subprocess.call(["bash", ftp_script, username, password, ftp, folder_to_move, src,folder_name_in_destination])
ftp_script:
lftp -u $1,$2 $3 <<EOF
mkdir $4
lcd $5
cd $6
mirror --reverse
EOF
Yet another answer using ftputil. And pathlib for figuring out the local path.
I've compiled and improved some of the above answers.
So thank you for the inspiration!
import os
import ftplib
import ftputil
from pathlib import Path
def ftp_upload(ftp_host, src_path, dst_path=None):
if dst_path is None:
ftp_host.path.join(os.curdir)
# Make sure that local path is a directory
if src_path.is_dir():
# Make sure that the root path exists
if ftp_host.path.exists(dst_path) is False:
# Inform user
print(f'mkdir: {src_path} -> {dst_path}')
# Create folder
ftp_host.mkdir(dst_path)
# Parse content of the root folder
for src_child in src_path.iterdir():
if src_child.is_dir():
# Use recursion for sub folders/files
dst_child = ftp_host.path.join(dst_path, src_child.stem)
ftp_upload(ftp_host, src_child, dst_child)
else:
# Copy sub files
dst_child = ftp_host.path.join(dst_path, src_child.name)
# Inform user
print(f'upload: {src_child} -> {dst_child}')
# Perform copy (upload)
ftp_host.upload(src_child, dst_child)
# ftp_host.upload_if_newer(src_child, dst_child)
else:
# Inform user
print(f"src_path: '{src_path}' must be an existing directory!")
if __name__ == '__main__':
# FTP settings
user = "ftp_user"
password = "ftp_password"
host = "localhost"
port = 2121
# Path settings
src_path = Path(__file__).parent.resolve() / "upload_dir"
dst_dir = "upload_dir"
# Inform user
print(f"Establishing FTP session as '{user}'")
# Establish FTP session
ftplib.FTP.port = port
with ftputil.FTPHost(host, user, password) as ftp_host:
# Perform recursive FTP upload
dst_path = ftp_host.path.join(os.curdir, dst_dir)
ftp_upload(ftp_host, src_path, dst_path)