I have a list of ftp sites ( eg:10 ) in text file and i need to download the last created file from ftp sites. Is this possible. This is my code :
import os
from ftplib import FTP
ftp = FTP("xxx.xx.xx.xx1", "USERNAME1", "PASSWORD1")
ftp = FTP("xxx.xx.xx.xx2", "USERNAME2", "PASSWORD2")
ftp = FTP("xxx.xx.xx.xx3", "USERNAME3", "PASSWORD3")
ftp = FTP("xxx.xx.xx.xx4", "USERNAME4", "PASSWORD4")
ftp = FTP("xxx.xx.xx.xx5", "USERNAME5", "PASSWORD5")
ftp.login()
ftp.retrlines("LIST")
ftp.cwd("SmythIN/2014-10-29") --- here i have a folder created by current date ...how can i pass current date folder i change directory.
ftp.cwd("subFolder") # or ftp.cwd("folderOne/subFolder")
listing = []
ftp.retrlines("LIST", listing.append)
words = listing[0].split(None, 8)
filename = words[-1].lstrip()
# download the file
local_filename = os.path.join(r"c:\myfolder", filename)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write, 8*1024)
lf.close()
updated code :
ftp.cwd("SmythIN/2014-10-29")- the directory with today date is already created.
Just looping through the servers and pulling the last file within specified directories (if I understand your question correctly) is straight forward. Remembering what server each file came from should not be problematic either since you can use different local directories on your local machine or edit the filename as the file transfers. Here is my suggestions (to be modified to your application of course):
import os
from ftplib import FTP
# read in text file containing server login information and jam into dictionary
with open('server_file.txt','r') as tmp:
servers = {}
for r in tmp.read().split('\n'):
rs = r.split(',') # split r by comma
servers[rs[0]] = {'uname':rs[1],'pwd':[rs[2]]}
# if you want to create a new directory to save the file to
heute = dt.datetime.strftime(dt.datetime.today(),'%Y%m%d')
if os.path.isdir('my_dir' + heute)==False:
os.mkdir('my_dir' + heute)
for s in servers:
ftp = FTP(s,servers[s]['uname'],servers[s]['pwd'])
ftp.cwd('desired_subdir')
# if you want to download the last file I would us nlst
with open('local_file','wb') as lf:
ftp.retrbinary('RETR' + ftp.nlst()[-1], lf.write, 8*1024)
Related
I need to create an script that helps me pick the URL from a Box file that has a path in my computer.
I've the Box desktop application installed.
Like: C:\Users\Thiago.Oliveira\Box\
I've created this script:
# Providing the folder path
origin = r'C:\\Users\\Thiago.Oliveira\\Box\\XXXXXXX.xlsx'
target = f'C:\\Users\\Thiago.Oliveira\\Box\\XXXXXXX{datetime.date.today()}.xlsx'
# Fetching the list of all the files
shutil.copy(origin, target)
print("Files are copied successfully")
This helps me out to copy and rename the file for this box folder. But I also want to pick up the URL from the newly created file so I can send it over in an e-mail.
I haven't found anything that would help me with this.
Is this possible?
Yes, you can, see the example below.
This example is using the box python sdk, and this is a JWT auth application script.
After uploading the file, the box sdk returns a file object, which has many properties.
I'm not sure what you mean by "pick up the URL", it could be the direct download URL of the file or a shared link.
The example is for the direct download URL.
To get the destination FOLDER_ID, you can look at the browser URL when you open the folder on the box.com app.
e.g.:image of demo folder url in browser
from datetime import date
import os
from boxsdk import JWTAuth, Client
def main():
auth = JWTAuth.from_settings_file(".jwt.config.json")
auth.authenticate_instance()
client = Client(auth)
folder_id = "163422716106"
user = client.user().get()
print(f"User: {user.id}:{user.name}")
folder = client.folder(folder_id).get()
print(f"Folder: {folder.id}:{folder.name}")
with open("./main.py", "rb") as file:
basename = os.path.basename(file.name)
file_name = os.path.splitext(basename)[0]
file_ext = os.path.splitext(basename)[1]
file_time = date.today().strftime("%Y%m%d")
box_file_name = file_name + "_" + file_time + file_ext
print(f"Uploading {box_file_name} to {folder.name}")
box_file = folder.upload_stream(file, box_file_name)
box_file.get()
print(f"File: {box_file.id}:{box_file.name}")
print(f"Download URL: {box_file.get_download_url()}")
if __name__ == "__main__":
main()
print("Done")
Resulting in:
User: 20344589936:UI-Elements-Sample
Folder: 163422716106:Box UI Elements Demo
Uploading main_20230203.py to Box UI Elements Demo
File: 1130939599686:main_20230203.py
Download URL: https://public.boxcloud.com/d/1/b1!5CgJ...fLc./download
I have a requirement where I have to pull the latest files from an FTP folder, the problem is that the filename is having spaces and the filename is having a specific pattern.
Below is the code I have implemented:
import sys
from ftplib import FTP
import os
import socket
import time
import pandas as pd
import numpy as np
from glob import glob
import datetime as dt
from __future__ import with_statement
ftp = FTP('')
ftp.login('','')
ftp.cwd('')
ftp.retrlines('LIST')
filematch='*Elig.xlsx'
downloaded = []
for filename in ftp.nlst(filematch):
fhandle=open(filename, 'wb')
print 'Getting ' + filename
ftp.retrbinary('RETR '+ filename, fhandle.write)
fhandle.close()
downloaded.append(filename)
ftp.quit()
I understand that I can append an empty list to ftp.dir() command, but since the filename is having spaces, I am unable to split it in the right way and pick the latest file of the type that I have mentined above.
Any help would be great.
You can get the file mtime by sending the MDTM command iff the FTP server supports it and sort the files on the FTP server accordingly.
def get_newest_files(ftp, limit=None):
"""Retrieves newest files from the FTP connection.
:ftp: The FTP connection to use.
:limit: Abort after yielding this amount of files.
"""
files = []
# Decorate files with mtime.
for filename in ftp.nlst():
response = ftp.sendcmd('MDTM {}'.format(filename))
_, mtime = response.split()
files.append((mtime, filename))
# Sort files by mtime and break after limit is reached.
for index, decorated_filename in enumerate(sorted(files, reverse=True)):
if limit is not None and index >= limit:
break
_, filename = decorated_filename # Undecorate
yield filename
downloaded = []
# Retrieves the newest file from the FTP server.
for filename in get_newest_files(ftp, limit=1):
print 'Getting ' + filename
with open(filename, 'wb') as file:
ftp.retrbinary('RETR '+ filename, file.write)
downloaded.append(filename)
The issue is that the FTP "LIST" command returns text for humans, which format depends on the FTP server implementation.
Using PyFilesystem (in place of the standard ftplib) and its API will provide a "list" API (search "walk") that provide Pythonic structures of the file and directories lists hosted in the FTP server.
http://pyfilesystem2.readthedocs.io/en/latest/index.html
I have this FTP with folder and it contains these files:
pw201602042000.nc,
pw201602042010.nc,
pw201602042020.nc,
pw201602042030.nc,
pw201602042040.nc,
pw201602042050.nc,
pw201602042100.nc,
pw201602042110.nc,
pw201602042120.nc,
pw201602042130.nc,
pw201602042140.nc,
pw201602042150.nc,
pw201602042200.nc
how to download only file ending with 00?
from ftplib import FTP
server = FTP("ip/serveradress")
server.login("user", "password")
server.retrlines("LIST")
server.cwd("Folder")
server.sendcmd("TYPE i") # ready for file transfer
server.retrbinary("RETR %s"%("pw201602042300.nc"), open("pw", "wb").write)
when you obtained the list of files as list_of_files, just use fnmatch to match the file names according to wildcard:
list_of_files = server.retrlines("LIST")
dest_dir = "."
for name in list_of_files:
if fnmatch.fnmatch(name,"*00.nc"):
with open(os.path.join(dest_dir,name), "wb") as f:
server.retrbinary("RETR {}".format(name), f.write)
(note that you're writing the files on the same "pw" output file, I changed that, reusing the original name and provided a destination directory variable, and protecting the open in a with block to ensure file is closed when exiting the block)
This may be a stupid question. My code works fine and says that I've downloaded each file from the FTP server, but when I go to the folder on my computer where my code is the downloaded files are not there. It has worked in the past but all of sudden I dont know where my files are. What is the problem? Thanks for any help
#Extract UV files from FTP server
import ftplib
try:
# Connection information
server = 'ftp.ncep.noaa.gov'
username = 'anonymous'
password = 'anything'
# Directory and matching information
directory = '/pub/data/nccf/com/hourly/prod/uv.20130729/'#Format:YearMonthDay Remeber to change date
filematch = '*.grib2'
# Establish the connection
ftp = ftplib.FTP(server)
ftp.login(username, password)
# Change to the proper directory
ftp.cwd(directory)
# Loop through matching files and download each one individually
for filename in ftp.nlst(filematch):
fhandle = open(filename, 'wb')
print 'Getting ' + filename
ftp.retrbinary('RETR ' + filename, fhandle.write)
fhandle.close()
except ftplib.all_errors as err:
print 'Change date on directory'
print err
I want to move a large number of files from a windows system to a unix ftp server using python. I have a csv which has the current full path and filename and the new base bath to send it to (see here for an example dataset).
I have got a script using os.renames to do the transfer and directory creation in windows but can figure out a way to easily do it via ftp.
import os, glob, arcpy, csv, sys, shutil, datetime
top=os.getcwd()
RootOutput = top
startpath=top
FileList = csv.reader(open('FileList.csv'))
filecount=0
successcount=0
errorcount=0
# Copy/Move to FTP when required
ftp = ftplib.FTP('xxxxxx')
ftp.login('xxxx', 'xxxx')
directory = '/TransferredData'
ftp.cwd(directory)
##f = open(RootOutput+'\\Success_LOG.txt', 'a')
##f.write("Log of files Succesfully processed. RESULT of process run #:"+str(datetime.datetime.now())+"\n")
##f.close()
##
for File in FileList:
infile=File[0]
# local network ver
#outfile=RootOutput+File[4]
#os.renames(infile, outfile)
# ftp netowrk ver
# outfile=RootOutput+File[4]
# ftp.mkd(directory)
print infile, outfile
I tried the process in http://forums.arcgis.com/threads/17047-Upload-file-to-FTP-using-Python-ftplib but this is for moving all files in a directory, I have the old and new full file names and just need it to create the intermediate directories.
Thanks,
The following might work (untested):
def mkpath(ftp, path):
path = path.rsplit('/', 1)[0] # parent directory
if not path:
return
try:
ftp.cwd(path)
except ftplib.error_perm:
mkpath(ftp, path)
ftp.mkd(path)
ftp = FTP(...)
directory = '/TransferredData/'
for File in FileList:
infile = File[0]
outfile = File[4].split('\\') # need forward slashes in FTP
outfile = directory + '/'.join(outfile)
mkpath(ftp, outfile)
ftp.storbinary('STOR '+outfile, open(infile, 'rb'))