Develop Reference

Python 3: urlextract package, PermissionError

I am using Windows 10 x64, with Python 3.6.1 x86.
I have this script from a few months ago which was working fine, but right now it gives me a weird error. The script is a simple one that extract URLs from tweets saved in .csv files.
This is the script:
import datetime
from urlextract import URLExtract
twitter_files_list = ['File1.csv', 'File2.csv', 'File3.csv']
input_path = my_path
# Find domain of URL
def find_domain(url):
return url.split("//")[-1].split("/")[0]
# Clean domain from useless chars
def clean_domain(domain):
domain = domain.replace("[", "")
domain = domain.replace("]", "")
domain = domain.replace("\'", "")
return domain
# Extract URLs from Tweets
def url_extract(filename):
print('\n' + filename + ':')
url_counter = 0
url_file = open('extracted_urls/urls_' + filename, 'a')
# Open file
f = open(input_path + filename, "r", encoding="utf8")
lines = f.readlines()
# Search for contents of column "text"
text = []
for x in lines:
text.append(x.split('\t')[4])
# Close file
f.close()
extractor = URLExtract()
for i in range(len(text)):
try:
if extractor.find_urls(text[i]): # Check if URL exists
url = extractor.find_urls(text[i])
domain = find_domain(str(url))
if not " " in domain:
url_file.write(str(clean_domain(domain)) + "\n")
url_counter += 1
except 'Not Found':
continue
url_file.close()
# Main
if __name__ == '__main__':
print('\nURL Characterization:\n')
# Start timer
start = datetime.datetime.now()
# Find the unique usernames for every file
for twitter_file in twitter_files_list:
print('Searching ' + str(twitter_file) + '...')
url_extract(twitter_file)
# End timer
end = datetime.datetime.now()
# Print results
print("\nProcess finished")
print("Total time: " + str(end - start))
This gives me the following error:
Traceback (most recent call last):
File "C:/Users/Aventinus/url_analysis/url_extractor.py", line 77, in <module>
url_extract(twitter_file)
File "C:/Users/Aventinus/url_analysis/url_extractor.py", line 50, in url_extract
extractor = URLExtract()
File "C:\Program Files (x86)\Python36-32\lib\site-packages\urlextract.py", line 65, in __init__
if not self._download_tlds_list():
File "C:\Program Files (x86)\Python36-32\lib\site-packages\urlextract.py", line 114, in _download_tlds_list
with open(self._tld_list_path, 'w') as ftld:
PermissionError: [Errno 13] Permission denied: 'C:\\Program Files (x86)\\Python36-32\\lib\\site-packages\\.tlds'
I have no idea how to interpret this.

you can try run the script as administrator

IOError: [Errno 2] No such file or directory while using pysftp

I am trying to do a script to find files that contain a text string. The files are on a remote host but when I run it, I get an error
Traceback (most recent call last): File "dirsearch.py", line 55, in
fo = open(search_path + fname) IOError: [Errno 2] No such file or directory: u'/home/black/white/goto/reports/dbs-01-Apr-2017.log'
The script I used is below.
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from builtins import input
from builtins import open
from future import standard_library
standard_library.install_aliases()
import pysftp
#connect to sftp server
srv = pysftp.Connection(host="host", username="username",
password="password")
#acess remote directory on server
search_path = ('/home/black/white/goto/reports/')
file_type = '.log'
search_str = input("Enter the search string : ")
#addition ........or fname in os.listdir(path=search_path):
for fname in srv.listdir(search_path):
# Apply file type filter
if fname.endswith(file_type):
# Open file for reading
fo = open(search_path + fname)
# Read the first line from the file
line = fo.readline()
# Initialize counter for line number
line_no = 1
# Loop until EOF
while line != '' :
# Search for string in line
index = line.find(search_str)
if ( index != -1) :
print(fname, "[", line_no, ",", index, "] ", line, sep="")
# Read next line
line = fo.readline()
# Increment line counter
line_no += 1
# Close the files
fo.close()
srv.close()

decrypting function is lauching error at the end

I want to create a decrypt script in python, my function filename = allfiles(); return the path where is the files , but I try to decrypt my files launch me the error like this below. How can I fix it? I am using python 2.7
Traceback (most recent call last):
File "F:\bug_bounty\decrypt.py", line 41, in <module>
if os.path.basename(filename).startswith("(encrypted)"):
File "C:\Python27\lib\ntpath.py", line 208, in basename
return split(p)[1]
File "C:\Python27\lib\ntpath.py", line 180, in split
d, p = splitdrive(p)
File "C:\Python27\lib\ntpath.py", line 116, in splitdrive
normp = p.replace(altsep, sep)
AttributeError: 'list' object has no attribute 'replace'
Code:
from Crypto.Hash import SHA256
from Crypto.Cipher import AES
import os
import random
import sys
def decrypt(key, filename):
outFile = os.path.join(os.path.dirname(filename),
os.path.basename(filename[11:]))
chunksize = 64 * 1024
with open(filename, 'rb') as infile:
filesize = infile.read(16)
IV = infile.read(16)
decryptor = AES.new(key, AES.MODE_CBC, IV)
with open(outFile, 'wb') as outfile:
while True:
chunk = infile.read(chunksize)
if len(chunk) == 0:
break
outfile.write(decryptor.decrypt(chunk))
outfile.truncate(int(filesize))
def allfiles():
allFiles = []
for (root, subfiles, files) in os.walk(os.getcwd()):
for names in files:
allFiles.append(os.path.join(root, names))
return allFiles
password = 'M4st3rRul3zs'
filename = allfiles();
for files in filename:
if os.path.basename(filename).startswith("(encrypted)"):
print "%s is already encrypted" %filename
pass
else:
decrypt(SHA256.new(password).digest(), filename)
print "Done decrypting %s" %filename
"""os.remove(filename)"""

Django StreamingHttpResponse deleting file after user finishes downloading it

I am having problem tracking when a file download has been completed when StreamingHttpResponse is used. My intention is to delete the file once it has been downloaded by the user.
Doing the following returned an exception in terminal killing the server.
def down(request, file_name):
if request.method == 'GET':
if file_name:
import os
fh = get_object_or_404(FileHandler, filename=file_name)
csv_path = os.path.join(fh.path, fh.filename)
csv_file = open(csv_path)
response = StreamingHttpResponse(csv_file, content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="{}"'.format(fh.filename)
csv_file.close()
# I can now delete the file using os.remove(csv_path). Not sure since response has not been returned
return response
return HttpResponseRedirect('/b2b/export/')
Trace back:
----------------------------------------
Exception happened during processing of request from ('127.0.0.1', 59899)
Traceback (most recent call last):
File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 599, in process_request_thread
self.finish_request(request, client_address)
File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 334, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/Users/Michael/.virtualenvs/scrape/lib/python2.7/site-packages/django/core/servers/basehttp.py", line 102, in __init__
super(WSGIRequestHandler, self).__init__(*args, **kwargs)
File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 655, in __init__
self.handle()
File "/Users/Michael/.virtualenvs/scrape/lib/python2.7/site-packages/django/core/servers/basehttp.py", line 182, in handle
handler.run(self.server.get_app())
File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/wsgiref/handlers.py", line 92, in run
self.close()
File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/wsgiref/simple_server.py", line 33, in close
self.status.split(' ',1)[0], self.bytes_sent
AttributeError: 'NoneType' object has no attribute 'split'
----------------------------------------
What works is as follows but am not sure when to delete the file or know when download has been completed. Most importantly how to close the file:
def down(request, file_name):
if request.method == 'GET':
if file_name:
import os
fh = get_object_or_404(FileHandler, filename=file_name)
csv_path = os.path.join(fh.path, fh.filename)
response = StreamingHttpResponse(open(csv_path), content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="{}"'.format(fh.filename)
return response
return HttpResponseRedirect('/b2b/export/')

I have tried the above suggestion by #mwag; however with FileWrapper.
In my case, wanted to zip up directories and get rid of the archive when download is done.
import os, time, zipfile
from django.http import StreamingHttpResponse
from wsgiref.util import FileWrapper
class FileDeleteWrapper(FileWrapper):
def __init__(self, filepath, *args, **kwargs):
self.filepath = filepath
super(FileDeleteWrapper, self).__init__(*args, **kwargs)
def __del__(self, *args, **kwargs):
os.remove(self.filepath)
# View function
def zipFiles(request, assetId):
asset = get_object_or_404(Asset, id=assetId)
try:
files = File.objects.filter(asset=asset)
prefix = str(time.time()) +'_'
zipPath = os.path.join(
settings.ZIPPED_FILES_DIR,
prefix + asset.label+'.zip'
)
z = zipfile.ZipFile(zipPath, 'w', zipfile.ZIP_DEFLATED)
for f in files:
path = os.path.join(
mainSet.MEDIA_ROOT,
str(f.file)
)
z.write(path, str(f))
z.close()
chunkSize = 16384
response = StreamingHttpResponse(
FileDeleteWrapper(
filepath = zipPath,
filelike=open(zipPath, 'rb'),
blksize=chunkSize
)
)
response['Content-Length'] = os.path.getsize(zipPath)
response['Content-Disposition'] = "attachment; filename=%s" % asset.label+'.zip'
return response
except Exception as e:
if mainSet.DEBUG:
print(type(e))
else:
# log expception
raise Http404

Try creating a class that will delete the file when it is gc'd. For example, something like the below might work:
class open_then_delete(object):
def __init__(self, filename, mode='rb'):
self.filename = filename
self.file_obj = open(filename, mode)
def __del__(self):
self.close()
def close(self):
if self.file_obj:
self.file_obj.close()
self.file_obj = None
self.cleanup()
def cleanup(self):
if self.filename:
try:
sys.stderr.write('open_then_delete: del ' + self.filename)
os.remove(self.filename)
except:
pass
self.filename = None
def __getattr__(self, attr):
return getattr(self.file_obj, attr)
def __iter__(self):
return iter(self.file_obj)
# below is your code, modified to use use_then_delete
def down(request, file_name):
if request.method == 'GET':
if file_name:
import os
fh = get_object_or_404(FileHandler, filename=file_name)
csv = open_then_delete(os.path.join(fh.path, fh.filename))
response = StreamingHttpResponse(csv.open(), content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="{}"'.format(fh.filename)
return response
return HttpResponseRedirect('/b2b/export/')

Python error extracting some zip files

i wrote a small app that downloads a zip file (with a different extension) when provided by a link and extracts the file to a renamed folder.
For some reason its working for some of my zip files, but not for all of them.
I get a :
Traceback (most recent call last):
File "download_unzip.py", line 48, in <module>
main()
File "download_unzip.py", line 42, in main
shutil.move(unzip_file(temp_kmz),'temp_extracted/')
File "download_unzip.py", line 26, in unzip_file
fd = open(name, 'w')
IOError: [Errno 2] No such file or directory: 'models/model.dae'
My code is :
import sys , urllib , zipfile , os.path , argparse , shutil
parser = argparse.ArgumentParser(description="Download and Unzip")
parser.add_argument('url', help='The action to take (e.g. install, remove, etc.)')
args = parser.parse_args()
print args.url
url = args.url
temp_kmz="temp_kmz"
def unzip_file(path):
zfile = zipfile.ZipFile(path)
extracted_filename = zfile.infolist()[0].filename[:-1]
for name in zfile.namelist():
(dirname, filename) = os.path.split(name)
#print "Decompressing " + filename + " on " + dirname
if filename == '':
# directory
if not os.path.exists(dirname):
os.mkdir(dirname)
else:
# file
fd = open(name, 'w')
fd.write(zfile.read(name))
fd.close()
zfile.close()
return extracted_filename
def download_file():
urllib.urlretrieve (url, temp_kmz)
return True
def main():
if (download_file()):
print "Now deleting temp..."
shutil.rmtree('temp_extracted/')
print "unzipping.. and renaming folder"
shutil.move(unzip_file(temp_kmz),'temp_extracted/')
print "Finished!!"
else:
print "Error downloading file"
main()
my working downloaded file:
python download_unzip.py "http://dl.dropbox.com/u/2971439/dae.kmz"
The one that is not working:
python download_unzip.py
"http://dl.dropbox.com/u/2971439/rally_car_youbeq.kmz"
Please note that both files extract properly with my OS (Ubuntu)

fixed my problem with some heavy code changes:
import urllib2 ,argparse, shutil, urlparse , os , zipfile, os.path
from zipfile import ZipFile as zip
parser = argparse.ArgumentParser(description="Download and Unzip")
parser.add_argument('url', help='The action to take (e.g. install, remove, etc.)')
args = parser.parse_args()
print args.url
url = args.url
temp_kmz="temp_kmz"
def extractAll(zipName):
z = zip(zipName)
for f in z.namelist():
if f.endswith('/'):
os.makedirs(f)
else:
z.extract(f)
def download(url, fileName=None):
def getFileName(url,openUrl):
if 'Content-Disposition' in openUrl.info():
# If the response has Content-Disposition, try to get filename from it
cd = dict(map(
lambda x: x.strip().split('=') if '=' in x else (x.strip(),''),
openUrl.info()['Content-Disposition'].split(';')))
if 'filename' in cd:
filename = cd['filename'].strip("\"'")
if filename: return filename
# if no filename was found above, parse it out of the final URL.
return os.path.basename(urlparse.urlsplit(openUrl.url)[2])
r = urllib2.urlopen(urllib2.Request(url))
try:
fileName = fileName or getFileName(url,r)
with open(fileName, 'wb') as f:
shutil.copyfileobj(r,f)
finally:
r.close()
def main():
download(url,temp_kmz)
extractAll(temp_kmz)
main()