Python newb...beware!
I am trying to recursively ftp some files. In the script below, I get an error:
Traceback (most recent call last):
File "dump.py", line 7, in <module>
for root,dirs,files in recursive:
File "/usr/local/lib/python2.6/site-packages/ftputil/ftputil.py", line 880, in walk
if self.path.isdir(self.path.join(top, name)):
File "/usr/local/lib/python2.6/site-packages/ftputil/ftp_path.py", line 133, in isdir
path, _exception_for_missing_path=False)
File "/usr/local/lib/python2.6/site-packages/ftputil/ftputil.py", line 860, in stat
return self._stat._stat(path, _exception_for_missing_path)
File "/usr/local/lib/python2.6/site-packages/ftputil/ftp_stat.py", line 624, in _stat
_exception_for_missing_path)
File "/usr/local/lib/python2.6/site-packages/ftputil/ftp_stat.py", line 578, in __call_with_parser_retry
result = method(*args, **kwargs)
File "/usr/local/lib/python2.6/site-packages/ftputil/ftp_stat.py", line 543, in _real_stat
lstat_result = self._real_lstat(path, _exception_for_missing_path)
File "/usr/local/lib/python2.6/site-packages/ftputil/ftp_stat.py", line 502, in _real_lstat
for stat_result in self._stat_results_from_dir(dirname):
File "/usr/local/lib/python2.6/site-packages/ftputil/ftp_stat.py", line 419, in _stat_results_from_dir
lines = self._host_dir(path)
File "/usr/local/lib/python2.6/site-packages/ftputil/ftp_stat.py", line 411, in _host_dir
return self._host._dir(path)
File "/usr/local/lib/python2.6/site-packages/ftputil/ftputil.py", line 811, in _dir
descend_deeply=True)
File "/usr/local/lib/python2.6/site-packages/ftputil/ftputil.py", line 578, in _robust_ftp_command
self.chdir(path)
File "/usr/local/lib/python2.6/site-packages/ftputil/ftputil.py", line 603, in chdir
ftp_error._try_with_oserror(self._session.cwd, path)
File "/usr/local/lib/python2.6/site-packages/ftputil/ftp_error.py", line 146, in _try_with_oserror
raise PermanentError(*exc.args)
ftputil.ftp_error.PermanentError: 550 /usr/local/web: No such file or directory
Debugging info: ftputil 2.6, Python 2.6.5 (linux2)
Exited: 256
I have no idea what that means. However, when I change the directory to "/path/dir2" it works and prints out "file.txt".
Here's my directory structures:
/path/dir1/another/file.txt # I get the above error with this path
/path/dir2/another/file.txt # this works fine, even though the directories have the same structure
My script:
import ftputil
ftp = ftputil.FTPHost('ftp.site.com','user','pass')
recursive = ftp.walk("/path/dir1",topdown=True,onerror=None)
for root,dirs,files in recursive:
for name in files:
print name
ftp.close
Is there a symblolic link in /path/dir1/ that points back to /usr/local/web?
If you want to step over this error you could put a try catch block in and log...
ftp = ftputil.FTPHost('ftp.site.com','user','pass')
try:
recursive = ftp.walk("/path/dir1",topdown=True,onerror=None)
for root,dirs,files in recursive:
for name in files:
print name
except Error e:
print "Error: %s occurred" % (e)
ftp.close
the above will catch all errors. you can also import the specific error your getting and do this...
import ftputil.ftp_error.PermanentError as PermanentError
ftp = ftputil.FTPHost('ftp.site.com','user','pass')
try:
recursive = ftp.walk("/path/dir1",topdown=True,onerror=None)
for root,dirs,files in recursive:
for name in files:
print name
except PermanentError e:
print "Permanent Error: %s occurred" % (e)
ftp.close
Related
i 'm trying to make a for loop who browse files in a specific directory while creating a folder if he doesn't exist with this solution. here is the code:
import ftputil
host=ftputil.FTPHost('x.x.x.x',"x","x") #connecting to the ftp server
mypathexist='./CameraOld' (he is here: /opt/Camera/CameraOld
mypath = '.' #it put you in /opt/Camera (it's the default path configured)
host.chdir(mypath)
files = host.listdir(host.curdir)
for f in files: #i browse the files in my folders
if f==mypathexist: #if a file is named CameraOld (it's a folder)
isExist=True
break
else: isExist=False #if 0 file are named like it
print(isExist)
if isExist==False: #if the file doesn't exist
host.mkdir(mypathexist) #create the folder
else:
print("ok")
The problem is that isExist is always false so the script try to create a folder who is already created. And i don't understand why.
Here's the output:
False #it's the print(isExist)
Traceback (most recent call last):
File "/usr/local/lib/python3.10/dist-packages/ftputil/host.py", line 695, in command
self._session.mkd(path)
File "/usr/lib/python3.10/ftplib.py", line 637, in mkd
resp = self.voidcmd('MKD ' + dirname)
File "/usr/lib/python3.10/ftplib.py", line 286, in voidcmd
return self.voidresp()
File "/usr/lib/python3.10/ftplib.py", line 259, in voidresp
resp = self.getresp()
File "/usr/lib/python3.10/ftplib.py", line 254, in getresp
raise error_perm(resp)
ftplib.error_perm: 550 CameraOld: file exist
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/user/Bureau/try.py", line 16, in <module>
host.mkdir(mypathexist)
File "/usr/local/lib/python3.10/dist-packages/ftputil/host.py", line 697, in mkdir
self._robust_ftp_command(command, path)
File "/usr/local/lib/python3.10/dist-packages/ftputil/host.py", line 656, in _robust_ftp_command
return command(self, tail)
File "/usr/local/lib/python3.10/dist-packages/ftputil/host.py", line 694, in command
with ftputil.error.ftplib_error_to_ftp_os_error:
File "/usr/local/lib/python3.10/dist-packages/ftputil/error.py", line 195, in __exit__
raise PermanentError(
ftputil.error.PermanentError: 550 CameraOld: file exist
Debugging info: ftputil 5.0.4, Python 3.10.4 (linux)
I would bet your mypathexist is not correct. Or the other way around, your file list, doesn't hold the strings in that condition you assume it does.
Take a look at your condition by hand. Print out f in your loop. Is it what you would expect to be?
In the end, Python is simply comparing Strings.
Got a read-only file within a zip file which are password protected and I need to extract it to the /tmp directory.
I get a CRC-32 error which suggests that the file would be corrupted yet I know it isn't and is in fact a read-only file. Any Suggestions?
Error:
Traceback (most recent call last):
File "/tmp/usercode.py", line 45, in <module>
zip.extractall('/tmp',pwd = "piso")
File "/usr/lib64/python2.7/zipfile.py", line 1040, in extractall
self.extract(zipinfo, path, pwd)
File "/usr/lib64/python2.7/zipfile.py", line 1028, in extract
return self._extract_member(member, path, pwd)
File "/usr/lib64/python2.7/zipfile.py", line 1084, in _extract_member
shutil.copyfileobj(source, target)
File "/usr/lib64/python2.7/shutil.py", line 49, in copyfileobj
buf = fsrc.read(length)
File "/usr/lib64/python2.7/zipfile.py", line 632, in read
data = self.read1(n - len(buf))
File "/usr/lib64/python2.7/zipfile.py", line 672, in read1
self._update_crc(data, eof=(self._compress_left==0))
File "/usr/lib64/python2.7/zipfile.py", line 647, in _update_crc
raise BadZipfile("Bad CRC-32 for file %r" % self.name)
zipfile.BadZipfile: Bad CRC-32 for file 'alien-12.txt'
Code:
# importing required modules
from zipfile import ZipFile
# specifying the zip file name
file_name = "/tmp/alien-12.zip"
# opening the zip file in READ mode
with ZipFile(file_name, 'r') as zip:
# printing all the contents of the zip file
zip.printdir()
# extracting all the files
print('Extracting all the files now...')
zip.extractall('/tmp',pwd = "piso")
print('Done!')
If I change the line of:
zip.extractall('/tmp',pwd = "piso")
then I get the error of:
IOError: [Errno 30] Read-only file system:
Then go on to try and fix it first by trying to output what is in the zip file.
zipfile.testzip() returns which then errors
Error:
RuntimeError: File alien-12.txt is encrypted, password required for extraction
I should just unzip RESULT folder inside of output.tgz archive
#!/usr/bin/python
import os, sys, tarfile, gzip
def unZip(self,file, filename,dire):
tar = tarfile.open(filename,"r:gz")
for file in tar.getmembers():
if file.name.startswith("RESULT"):
tar.extract(file,dire)
print 'Done.'
def main():
utilities = ZipUtil()
filename = 'output.tgz'
directory = str(sys.argv[1]);
path = os.path.join(directory, filename)
utilities.unZip(directory,path,directory)
main()
and I'm getting this error
File "/usr/lib64/python2.7/tarfile.py", line 1805, in getmembers
self._load() # all members, we first have to
File "/usr/lib64/python2.7/tarfile.py", line 2380, in _load
tarinfo = self.next()
File "/usr/lib64/python2.7/tarfile.py", line 2315, in next
self.fileobj.seek(self.offset)
File "/usr/lib64/python2.7/gzip.py", line 429, in seek
self.read(1024)
File "/usr/lib64/python2.7/gzip.py", line 256, in read
self._read(readsize)
File "/usr/lib64/python2.7/gzip.py", line 303, in _read
self._read_eof()
File "/usr/lib64/python2.7/gzip.py", line 342, in _read_eof
hex(self.crc)))
IOError: CRC check failed 0xccf8af55 != 0x43cb4d43L
There are many questions like this but still now answer for this proble. The file output.tgz is downloaded and it might be corrupted this is why I guess its not working. But is there anyway around to tell tar to open it. THnak a lot!
I don't understand why my script is failing with an out of memory error after it runs about 200 tiles. I have looked at the script and am not sure what it's writing to memory that is so large that it fails...
The script
import os, glob, arcpy, csv, sys, shutil, datetime
top = r'L:\Raster_Data' # Source directory for all files to check and process
#top = os.getcwd()
RootOutput = r'H:\Raster' # Directory to copy/move files to
FileList = csv.reader(open(r'P:\2013\508_Data_Split_MSC\Working\LiDAR_Tiles_index2_MSC.csv')) # list of files to use with unique name
RootDirDelete="L:\\" #repatative name of roor file path to remove from new directory name
filecount=0
successcount=0
errorcount=0
print "Working in: "+ top
list =[]
f = open(RootOutput+'\\Success_LOG.txt', 'a')
f.write("Log of files Succesfully processed. RESULT of process run #:"+str(datetime.datetime.now())+"\n")
f.close()
for File in FileList:
for root, dirs, files in os.walk(top, topdown=False):
for directory in dirs:
currentPath=os.path.join(root,directory)
os.chdir(currentPath)
#arcpy.env.workspace = currentPath
print os.getcwd()
lstFCs = glob.glob('*'+File[0]+'*.*')
print lstFCs
for fc in lstFCs:
OutPutDir=RootOutput+"\\"+str(os.getcwd()).strip(RootDirDelete)
filecount=filecount+1
if OutPutDir.endswith(".gdb"):
print "File in GDB...SKIPPING"
else:
try:
os.makedirs(OutPutDir)
print "Created: "+OutPutDir
except:
#pass
print "Unexpected error:", sys.exc_info()[0]
try:
fcIn=currentPath+"\\"+fc
fcOut=OutPutDir+"\\"+fc
successcount=successcount+1
print "Copied: "+fcIn+" to " +fcOut
shutil.copy2(fcIn,fcOut)
f = open(RootOutput+'\\Success_LOG.txt', 'a')
f.write(str(filecount)+","+str(successcount)+","+str(fcIn)+","+str(fcOut)+"\n")
f.close()
except:
#pass
print "Unexpected error:", sys.exc_info()[0]
errorcount=errorcount+1
f = open(RootOutput+'\\Error_LOG.txt', 'a')
f.write(str(filecount)+","+str(errorcount)+","+str(fcIn)+","+str(fcOut)+","+str(sys.exc_info()[0])+"\n")
f.close()
The error report.
L:\Raster_Data\Satellite_Imagery\Spot\Eastern_QLD_and_TRC_2010\Shapefiles\AUO_Australia
[]
Traceback (most recent call last):
File "<string>", line 73, in execInThread
File "C:\Program Files (x86)\PyScripter\Lib\rpyc.zip\rpyc\core\netref.py", line 196, in __call__
File "C:\Program Files (x86)\PyScripter\Lib\rpyc.zip\rpyc\core\netref.py", line 71, in syncreq
File "C:\Program Files (x86)\PyScripter\Lib\rpyc.zip\rpyc\core\protocol.py", line 431, in sync_request
File "C:\Program Files (x86)\PyScripter\Lib\rpyc.zip\rpyc\core\protocol.py", line 379, in serve
File "C:\Program Files (x86)\PyScripter\Lib\rpyc.zip\rpyc\core\protocol.py", line 337, in _recv
File "C:\Program Files (x86)\PyScripter\Lib\rpyc.zip\rpyc\core\channel.py", line 50, in recv
File "C:\Program Files (x86)\PyScripter\Lib\rpyc.zip\rpyc\core\stream.py", line 166, in read
EOFError: [Errno 10054] An existing connection was forcibly closed by the remote host
>>>
*** Remote Interpreter Reinitialized ***
>>>
Here's the error:
I have a script that connects to server and makes a local copy of the whole directory.
EOFerror occurs after exactly 50 files of any choice have been downloaded.
Can anyone please tell me, what is wrong with the script?
ERROR:
Traceback (most recent call last):
File "ftp.py", line 37, in <module>
ftp_walk(ftp)
File "ftp.py", line 17, in ftp_walk
currdir = ftp.pwd()[1:]
File "/usr/lib/python2.7/ftplib.py", line 574, in pwd
resp = self.sendcmd('PWD')
File "/usr/lib/python2.7/ftplib.py", line 244, in sendcmd
return self.getresp()
File "/usr/lib/python2.7/ftplib.py", line 210, in getresp
resp = self.getmultiline()
File "/usr/lib/python2.7/ftplib.py", line 196, in getmultiline
line = self.getline()
File "/usr/lib/python2.7/ftplib.py", line 186, in getline
if not line: raise EOFError
EOFError
SCRIPT:
#!/usr/bin/python
import ftplib
import sys
import os
import datetime
def ftp_walk(ftp):
dirs = ftp.nlst()
for item in (path for path in dirs if path not in ('.', '..')):
try:
ftp.cwd(item)
print datetime.datetime.now().strftime("%Y-%m-%d %H:%M")+' DIR: ', ftp.pwd()
ftp_walk(ftp)
ftp.cwd('..')
except Exception, e:
currdir = ftp.pwd()[1:]
if not os.path.exists(currdir): os.makedirs(currdir)
try:
with open(currdir+"/"+item, 'wb') as f:
def callback(data):
f.write(data)
ftp.retrbinary('RETR %s' % item, callback)
f.close()
print datetime.datetime.now().strftime("%Y-%m-%d %H:%M")+' RETR: '+ currdir+"/"+item
except Exception, e:
print e
ftp = ftplib.FTP("hhhhhhhhhhhhhh")
ftp.login("aaaaaaaa", "bbbbbbbbbbb")
ftp.sendcmd("TYPE I") #binary mode
ftp.set_pasv(True) # Trying Passive mode
ftp.cwd("public_html/eeeeeeee/rrrrrrrr/images")
ftp_walk(ftp)
ftp.quit()
Edit:
After manual update of ftplib for python 2.7:
Traceback (most recent call last):
File "ftp.py", line 29, in <module>
ftp = ftplib.FTP("something.com")
File "/usr/lib/python2.7/ftplib.py", line 114, in __init__
self.connect(host)
File "/usr/lib/python2.7/ftplib.py", line 150, in connect
self.file = self.sock.makefile('r', encoding=self.encoding)
TypeError: makefile() got an unexpected keyword argument 'encoding'
I have tried your script and it works without any problems. I just pulled 233 images from my server using it. Try setting the current dir to ftp.cwd("./public_html/eeeeeeee/rrrrrrrr/images") and see what happens...