Would like to get this script working with python3 (Python 3.10.4):
https://stackoverflow.com/a/2573715/2394635
It would be where it says Full code bellow:
I don't put the code directly because I get the stackoverflow notification It looks like your post is mostly code; please add some more details.
I've used the pip script 2to3, with this resulting code:
import sys, os, hashlib, io, bencode
def pieces_generator(info):
"""Yield pieces from download file(s)."""
piece_length = info['piece length']
if 'files' in info: # yield pieces from a multi-file torrent
piece = ""
for file_info in info['files']:
path = os.sep.join([info['name']] + file_info['path'])
print(path)
sfile = open(path.decode('UTF-8'), "rb")
while True:
piece += sfile.read(piece_length-len(piece))
if len(piece) != piece_length:
sfile.close()
break
yield piece
piece = ""
if piece != "":
yield piece
else: # yield pieces from a single file torrent
path = info['name']
print(path)
sfile = open(path.decode('UTF-8'), "rb")
while True:
piece = sfile.read(piece_length)
if not piece:
sfile.close()
return
yield piece
def corruption_failure():
"""Display error message and exit"""
print("download corrupted")
exit(1)
def main():
# Open torrent file
torrent_file = open(sys.argv[1], "rb")
metainfo = bencode.bdecode(torrent_file.read())
info = metainfo['info']
pieces = io.StringIO(info['pieces'])
# Iterate through pieces
for piece in pieces_generator(info):
# Compare piece hash with expected hash
piece_hash = hashlib.sha1(piece).digest()
if (piece_hash != pieces.read(20)):
corruption_failure()
# ensure we've read all pieces
if pieces.read():
corruption_failure()
if __name__ == "__main__":
main()
However, keeps failing:
% python3 extract-torrent.py archive.torrent
Traceback (most recent call last):
File "/home/smt/Documents/extract-torrent-py3.py", line 1, in <module>
import sys, os, hashlib, io, bencode
File "/home/smt/.local/lib/python3.10/site-packages/bencode.py", line 73, in <module>
from types import StringType, IntType, LongType, DictType, ListType, TupleType
ImportError: cannot import name 'StringType' from 'types' (/usr/lib/python3.10/types.py)
Any help?
As #9769953 pointed out, bencode is not compatible with Python 3.10. You could try bencodepy which claims to be compatible with both Python 2 and 3.
From the website:
Install with pip install bencode.py
Import with import bencodepy
i am new on python, i wrote simple script to send all image sorted by time in folder to API. this code working on just one file(jpg), and can't send the rest of image in folder. I want to if i run this code, it's just waiting until some image added to current folder, when image is inside folder then it will send to API by time based on images that first existed. I am very confused, any helps will be appriciated! thx
import glob
import argparse
import requests
import json
import time
import os
def main():
result = []
file = glob.glob("/path/to/dir/*.jpg")
regions = ['id']
time_to_wait = 10000
time_counter = 0
while not os.path.exists(file):
time.sleep(1)
time_counter += 1
if time_counter > time_to_wait: break
print("waiting for file...")
if os.path.isfile(file):
with open(file, 'rb') as fp:
response = requests.post(
'https://GET_API/',
data=dict(regions=regions),
files=dict(upload=fp),
headers={'Authorization': 'Token ' + 'XXX'})
result.append(response.json())
resp_dict = json.loads(json.dumps(result, indent=2))
if resp_dict[0]['results']:
num=resp_dict[0]['results'][0]['plate']
print(f"DETECTED NUMBER: {num}")
os.remove(file)
else:
print("file doesn't exists!")
if __name__ == '__main__':
main()
You didn't update your file on each iteration. Maybe that's why no more new files are detected. file also need to be treated as a list, so I guess you should iterate through file. Your while loop should look like this:
while True:
files = glob.glob(os.path.join('path', 'to', 'dir', '*.jpg'))
for file in files:
if os.path.isfile(file):
with open(file, 'rb') as fp:
# Upload and delete
# sleep
What is the best practice to try to open the file and retry after n seconds?
Currently, I do:
import os
from os import path
import shutil
dir_path = path.abspath(path.join("foo", "bar"))
destination_path = path.abspath(path.join("foo", "dest_dir"))
for f in dir_path:
try:
# try to open the file if the file isn't used any longer
file_opened = open(f, 'r')
# move the file on successful opening
shutil.move(file_opened, destination_path)
file_opened.close()
except IOError:
return False
So, at the moment I do not handle the exception. I think about creation of extra function to open the file and recall the function on excepth with time.sleep(n)
However, I am sure there must be something else ...
I do not use
with open(f, 'rb') as file_opened:
do whatever`
EDIT:
One process creates the file and I want Python process to move the file once I am sure the file writting / creation is completed. So, I have added shutil.move in the above code to show the whole situation.
EDIT:
Please find below the code I have developed to solve the problem. I ended with writing own custom solution to handle it:
import os
from os import path
import psutil
from retry import retry
import shutil
from subprocess import check_output,Popen, PIPE
import glob
import time
class FileHandler:
def __init__(self, fn_source, dir_source):
self.file_source_name = fn_source
self.file_source_path = path.join(dir_source, self.file_source_name)
self.dir_dest_path = path.join(dir_source, "test")
self.file_dest_path = path.join(self.dir_dest_path, self.file_source_name)
def check_file(self):
if os.path.exists(self.file_source_path):
try:
os.rename(self.file_source_path, self.file_source_path)
print("file renamed")
return True
except:
print("can not rename the file..retrying")
time.sleep(1)
self.check_file()
else:
print("source file does not exist...retrying")
time.sleep(5)
self.check_file()
def check_destination(self):
if os.path.exists(self.file_source_path) and not os.path.exists(self.file_dest_path):
return True
elif os.path.exists(self.file_source_path) and os.path.exists(self.file_dest_path):
try:
print(self.file_dest_path, self.file_source_name)
os.remove(self.file_dest_path)
return True
except Exception as e:
print("can not remove the file..retrying")
time.sleep(5)
self.check_destination()
def move_file(self):
if self.check_destination():
print(self.file_source_path)
shutil.move(self.file_source_path, self.file_dest_path)
print("moved", str(self.file_source_path))
return True
else:
print("can not move the file..retrying")
time.sleep(1)
self.move_file()
def file_ops(self):
if self.check_file():
self.move_file()
else:
print("source does not exist")
time.sleep(1)
self.file_ops()
return True
def each_file_ops(fn, dir_source):
fh = FileHandler(fn, dir_source)
return fh.file_ops()
def main(dir_source):
dir_files = glob.glob(path.join(dir_source, '*.txt'))
if dir_files:
[each_file_ops(f, dir_source) for f in dir_files]
else:
print("source dir is empty")
time.sleep(1)
main(dir_source)
if __name__ == '__main__':
main(path.join(""))
You can use the retry module for these kind of retrying. This makes the code to look much cleaner. pip install retry should install the module
from retry import retry
import shutil
#retry((FileNotFoundError, IOError), delay=1, backoff=2, max_delay=10, tries=100)
def attempt_to_move_file(fname, dest_path):
# Your attempt to move file
# shutil.move(fname, destination_path)
With the above code, when attempt_to_move_file is invoked it would be retried (upto a max of 100 tries) whenever we hit FileNotFoundError or IOError and the retry happens with a sleep 1, 2, 4, 8, 10, 10, 10 ... seconds between attempts
you do not need to open a file to move it. Because once you have opened a file it is in open state hence you can not move it, Take it like this, when you are playing a video in a player and you try to delete it or cut and paste system won't allow you.
import shutil
file_name = 'trytry.csv'
shutil.copyfile(file_name, '/Users/umeshkaushik/PycharmProjects/newtry.csv')
shutil.move(file_name, '/Users/umeshkaushik/PycharmProjects/newtry1.csv')
Above code runs fine. Just make sure you are providing correct input and output path.
I am trying to save jpegs to a file from a list of urls. This code times out frequently and randomly. It has saved up to 113 jpegs, there are many more than that, and sometimes only saves 10 before timing out. Is there a way to put a wait in so the timeout doesn't occur? I have tried sleep in the commented section with no luck. Thanks for the feedback!
Heres the timeout error message:
import urllib.request
import urllib
import codecs
from urllib import request
import time
import csv
class File:
def __init__(self, data):
self.data = data
file = File("1")
with open("file.csv", encoding = "utf8") as f1:
file.data = list(csv.reader(f1, skipinitialspace = True))
for i in file.data[1:]:
if len(i[27]) != 0:
#i[14] creates a unique jpeg file name in the dir
image = open('C:\\aPath'+i[14]+'.JPG', 'wb')
path = 'aPath' + i[14] + '.JPG'
#time.sleep(2) Tried sleep here, didn't work
#i[27] is a working jpeg url
urllib.request.urlretrieve(i[27], path)
image.close()
print('done!')
There's no way to prevent the exception. You need to catch the exception and retry.
...
for i in file.data[1:]:
if not i[27]:
continue
path = 'aPath' + i[14] + '.JPG'
while True: # retry loop
try:
urllib.request.urlretrieve(i[27], path)
break # On success, stop retry.
except TimeoutError:
print('timeout, retry in 1 second.')
time.sleep(1)
BTW, you don't need to open file if you use urllib.request.urlretrieve.
I know the URL of an image on Internet.
e.g. http://www.digimouth.com/news/media/2011/09/google-logo.jpg, which contains the logo of Google.
Now, how can I download this image using Python without actually opening the URL in a browser and saving the file manually.
Python 2
Here is a more straightforward way if all you want to do is save it as a file:
import urllib
urllib.urlretrieve("http://www.digimouth.com/news/media/2011/09/google-logo.jpg", "local-filename.jpg")
The second argument is the local path where the file should be saved.
Python 3
As SergO suggested the code below should work with Python 3.
import urllib.request
urllib.request.urlretrieve("http://www.digimouth.com/news/media/2011/09/google-logo.jpg", "local-filename.jpg")
import urllib
resource = urllib.urlopen("http://www.digimouth.com/news/media/2011/09/google-logo.jpg")
output = open("file01.jpg","wb")
output.write(resource.read())
output.close()
file01.jpg will contain your image.
I wrote a script that does just this, and it is available on my github for your use.
I utilized BeautifulSoup to allow me to parse any website for images. If you will be doing much web scraping (or intend to use my tool) I suggest you sudo pip install BeautifulSoup. Information on BeautifulSoup is available here.
For convenience here is my code:
from bs4 import BeautifulSoup
from urllib2 import urlopen
import urllib
# use this image scraper from the location that
#you want to save scraped images to
def make_soup(url):
html = urlopen(url).read()
return BeautifulSoup(html)
def get_images(url):
soup = make_soup(url)
#this makes a list of bs4 element tags
images = [img for img in soup.findAll('img')]
print (str(len(images)) + "images found.")
print 'Downloading images to current working directory.'
#compile our unicode list of image links
image_links = [each.get('src') for each in images]
for each in image_links:
filename=each.split('/')[-1]
urllib.urlretrieve(each, filename)
return image_links
#a standard call looks like this
#get_images('http://www.wookmark.com')
This can be done with requests. Load the page and dump the binary content to a file.
import os
import requests
url = 'https://apod.nasa.gov/apod/image/1701/potw1636aN159_HST_2048.jpg'
page = requests.get(url)
f_ext = os.path.splitext(url)[-1]
f_name = 'img{}'.format(f_ext)
with open(f_name, 'wb') as f:
f.write(page.content)
Python 3
urllib.request — Extensible library for opening URLs
from urllib.error import HTTPError
from urllib.request import urlretrieve
try:
urlretrieve(image_url, image_local_path)
except FileNotFoundError as err:
print(err) # something wrong with local path
except HTTPError as err:
print(err) # something wrong with url
I made a script expanding on Yup.'s script. I fixed some things. It will now bypass 403:Forbidden problems. It wont crash when an image fails to be retrieved. It tries to avoid corrupted previews. It gets the right absolute urls. It gives out more information. It can be run with an argument from the command line.
# getem.py
# python2 script to download all images in a given url
# use: python getem.py http://url.where.images.are
from bs4 import BeautifulSoup
import urllib2
import shutil
import requests
from urlparse import urljoin
import sys
import time
def make_soup(url):
req = urllib2.Request(url, headers={'User-Agent' : "Magic Browser"})
html = urllib2.urlopen(req)
return BeautifulSoup(html, 'html.parser')
def get_images(url):
soup = make_soup(url)
images = [img for img in soup.findAll('img')]
print (str(len(images)) + " images found.")
print 'Downloading images to current working directory.'
image_links = [each.get('src') for each in images]
for each in image_links:
try:
filename = each.strip().split('/')[-1].strip()
src = urljoin(url, each)
print 'Getting: ' + filename
response = requests.get(src, stream=True)
# delay to avoid corrupted previews
time.sleep(1)
with open(filename, 'wb') as out_file:
shutil.copyfileobj(response.raw, out_file)
except:
print ' An error occured. Continuing.'
print 'Done.'
if __name__ == '__main__':
url = sys.argv[1]
get_images(url)
A solution which works with Python 2 and Python 3:
try:
from urllib.request import urlretrieve # Python 3
except ImportError:
from urllib import urlretrieve # Python 2
url = "http://www.digimouth.com/news/media/2011/09/google-logo.jpg"
urlretrieve(url, "local-filename.jpg")
or, if the additional requirement of requests is acceptable and if it is a http(s) URL:
def load_requests(source_url, sink_path):
"""
Load a file from an URL (e.g. http).
Parameters
----------
source_url : str
Where to load the file from.
sink_path : str
Where the loaded file is stored.
"""
import requests
r = requests.get(source_url, stream=True)
if r.status_code == 200:
with open(sink_path, 'wb') as f:
for chunk in r:
f.write(chunk)
Using requests library
import requests
import shutil,os
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
}
currentDir = os.getcwd()
path = os.path.join(currentDir,'Images')#saving images to Images folder
def ImageDl(url):
attempts = 0
while attempts < 5:#retry 5 times
try:
filename = url.split('/')[-1]
r = requests.get(url,headers=headers,stream=True,timeout=5)
if r.status_code == 200:
with open(os.path.join(path,filename),'wb') as f:
r.raw.decode_content = True
shutil.copyfileobj(r.raw,f)
print(filename)
break
except Exception as e:
attempts+=1
print(e)
ImageDl(url)
Use a simple python wget module to download the link. Usage below:
import wget
wget.download('http://www.digimouth.com/news/media/2011/09/google-logo.jpg')
This is very short answer.
import urllib
urllib.urlretrieve("http://photogallery.sandesh.com/Picture.aspx?AlubumId=422040", "Abc.jpg")
Version for Python 3
I adjusted the code of #madprops for Python 3
# getem.py
# python2 script to download all images in a given url
# use: python getem.py http://url.where.images.are
from bs4 import BeautifulSoup
import urllib.request
import shutil
import requests
from urllib.parse import urljoin
import sys
import time
def make_soup(url):
req = urllib.request.Request(url, headers={'User-Agent' : "Magic Browser"})
html = urllib.request.urlopen(req)
return BeautifulSoup(html, 'html.parser')
def get_images(url):
soup = make_soup(url)
images = [img for img in soup.findAll('img')]
print (str(len(images)) + " images found.")
print('Downloading images to current working directory.')
image_links = [each.get('src') for each in images]
for each in image_links:
try:
filename = each.strip().split('/')[-1].strip()
src = urljoin(url, each)
print('Getting: ' + filename)
response = requests.get(src, stream=True)
# delay to avoid corrupted previews
time.sleep(1)
with open(filename, 'wb') as out_file:
shutil.copyfileobj(response.raw, out_file)
except:
print(' An error occured. Continuing.')
print('Done.')
if __name__ == '__main__':
get_images('http://www.wookmark.com')
Late answer, but for python>=3.6 you can use dload, i.e.:
import dload
dload.save("http://www.digimouth.com/news/media/2011/09/google-logo.jpg")
if you need the image as bytes, use:
img_bytes = dload.bytes("http://www.digimouth.com/news/media/2011/09/google-logo.jpg")
install using pip3 install dload
Something fresh for Python 3 using Requests:
Comments in the code. Ready to use function.
import requests
from os import path
def get_image(image_url):
"""
Get image based on url.
:return: Image name if everything OK, False otherwise
"""
image_name = path.split(image_url)[1]
try:
image = requests.get(image_url)
except OSError: # Little too wide, but work OK, no additional imports needed. Catch all conection problems
return False
if image.status_code == 200: # we could have retrieved error page
base_dir = path.join(path.dirname(path.realpath(__file__)), "images") # Use your own path or "" to use current working directory. Folder must exist.
with open(path.join(base_dir, image_name), "wb") as f:
f.write(image.content)
return image_name
get_image("https://apod.nasddfda.gov/apod/image/2003/S106_Mishra_1947.jpg")
this is the easiest method to download images.
import requests
from slugify import slugify
img_url = 'https://apod.nasa.gov/apod/image/1701/potw1636aN159_HST_2048.jpg'
img = requests.get(img_url).content
img_file = open(slugify(img_url) + '.' + str(img_url).split('.')[-1], 'wb')
img_file.write(img)
img_file.close()
If you don't already have the url for the image, you could scrape it with gazpacho:
from gazpacho import Soup
base_url = "http://books.toscrape.com"
soup = Soup.get(base_url)
links = [img.attrs["src"] for img in soup.find("img")]
And then download the asset with urllib as mentioned:
from pathlib import Path
from urllib.request import urlretrieve as download
directory = "images"
Path(directory).mkdir(exist_ok=True)
link = links[0]
name = link.split("/")[-1]
download(f"{base_url}/{link}", f"{directory}/{name}")
# import the required libraries from Python
import pathlib,urllib.request
# Using pathlib, specify where the image is to be saved
downloads_path = str(pathlib.Path.home() / "Downloads")
# Form a full image path by joining the path to the
# images' new name
picture_path = os.path.join(downloads_path, "new-image.png")
# "/home/User/Downloads/new-image.png"
# Using "urlretrieve()" from urllib.request save the image
urllib.request.urlretrieve("//example.com/image.png", picture_path)
# urlretrieve() takes in 2 arguments
# 1. The URL of the image to be downloaded
# 2. The image new name after download. By default, the image is saved
# inside your current working directory
Ok, so, this is my rudimentary attempt, and probably total overkill.
Update if needed, as this doesn't handle any timeouts, but, I got this working for fun.
Code listed here: https://github.com/JayRizzo/JayRizzoTools/blob/master/pyImageDownloader.py
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# =============================================================================
# Created Syst: MAC OSX High Sierra 21.5.0 (17G65)
# Created Plat: Python 3.9.5 ('v3.9.5:0a7dcbdb13', 'May 3 2021 13:17:02')
# Created By : Jeromie Kirchoff
# Created Date: Thu Jun 15 23:31:01 2022 CDT
# Last ModDate: Thu Jun 16 01:41:01 2022 CDT
# =============================================================================
# NOTE: Doesn't work on SVG images at this time.
# I will look into this further: https://stackoverflow.com/a/6599172/1896134
# =============================================================================
import requests # to get image from the web
import shutil # to save it locally
import os # needed
from os.path import exists as filepathexist # check if file paths exist
from os.path import join # joins path for different os
from os.path import expanduser # expands current home
from pyuser_agent import UA # generates random UserAgent
class ImageDownloader(object):
"""URL ImageDownloader.
Input : Full Image URL
Output: Image saved to your ~/Pictures/JayRizzoDL folder.
"""
def __init__(self, URL: str):
self.url = URL
self.headers = {"User-Agent" : UA().random}
self.currentHome = expanduser('~')
self.desktop = join(self.currentHome + "/Desktop/")
self.download = join(self.currentHome + "/Downloads/")
self.pictures = join(self.currentHome + "/Pictures/JayRizzoDL/")
self.outfile = ""
self.filename = ""
self.response = ""
self.rawstream = ""
self.createdfilepath = ""
self.imgFileName = ""
# Check if the JayRizzoDL exists in the pictures folder.
# if it doesn't exist create it.
if not filepathexist(self.pictures):
os.mkdir(self.pictures)
self.main()
def getFileNameFromURL(self, URL: str):
"""Parse the URL for the name after the last forward slash."""
NewFileName = self.url.strip().split('/')[-1].strip()
return NewFileName
def getResponse(self, URL: str):
"""Try streaming the URL for the raw data."""
self.response = requests.get(self.url, headers=self.headers, stream=True)
return self.response
def gocreateFile(self, name: str, response):
"""Try creating the file with the raw data in a custom folder."""
self.outfile = join(self.pictures, name)
with open(self.outfile, 'wb') as outFilePath:
shutil.copyfileobj(response.raw, outFilePath)
return self.outfile
def main(self):
"""Combine Everything and use in for loops."""
self.filename = self.getFileNameFromURL(self.url)
self.rawstream = self.getResponse(self.url)
self.createdfilepath = self.gocreateFile(self.filename, self.rawstream)
print(f"File was created: {self.createdfilepath}")
return
if __name__ == '__main__':
# Example when calling the file directly.
ImageDownloader("https://stackoverflow.design/assets/img/logos/so/logo-stackoverflow.png")
Download Image file, with avoiding all possible error:
import requests
import validators
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError
def is_downloadable(url):
valid=validators. url(url)
if valid==False:
return False
req = Request(url)
try:
response = urlopen(req)
except HTTPError as e:
return False
except URLError as e:
return False
else:
return True
for i in range(len(File_data)): #File data Contain list of address for image
#file
url = File_data[i][1]
try:
if (is_downloadable(url)):
try:
r = requests.get(url, allow_redirects=True)
if url.find('/'):
fname = url.rsplit('/', 1)[1]
fname = pth+File_data[i][0]+"$"+fname #Destination to save
#image file
open(fname, 'wb').write(r.content)
except Exception as e:
print(e)
except Exception as e:
print(e)