I am using Python 2.7.3 and I am trying to post data to my local web server. The data I am posting is temperature readings from my raspberry pi. I know the url is right because if I use the postman chrome plugin the data is successfully posted and I get a return message. In postman I can only use form-data though and NOT x-www-form-urlencoded which is how my python script has the content type setup. Can I change it to form-data?
Python Code:
import os
import glob
import time
import threading
import urllib
import urllib2
os.system('modprobe w1-gpio')
os.system('modprobe w1-therm')
base_dir = '/sys/bus/w1/devices/'
device_folder = glob.glob(base_dir + '28*')[0]
device_file = device_folder + '/w1_slave'
def read_temp_raw():
f = open(device_file, 'r')
lines = f.readlines()
f.close()
return lines
def read_temp():
lines = read_temp_raw()
while lines[0].strip()[-3:] != 'YES':
time.sleep(0.2)
lines = read_temp_raw()
equals_pos = lines[1].find('t=')
if equals_pos != -1:
temp_string = lines[1][equals_pos+2:]
temp_c = float(temp_string) / 1000.0
temp_f = temp_c * 9.0 / 5.0 + 32.0
temperature = {'tempf':temp_f, 'tempc':temp_c}
return temperature
def post():
threading.Timer(1800.0, post).start()
temperature = read_temp()
data = temperature
data['room'] = 'Server Room'
print(data)
data=urllib.urlencode(data)
path='http://client.pathtophppage' #the url you want to POST to
req=urllib2.Request(path, data)
req.add_header("Content-type", "application/x-www-form-urlencoded")
page=urllib2.urlopen(req).read()
post()
And the Error:
pi#raspberrypi ~/Documents $ python Temperature.py
{'tempc': 22.0, 'tempf': 71.6, 'room': 'Server Room'}
Traceback (most recent call last):
File "Temperature.py", line 49, in <module>
post()
File "Temperature.py", line 45, in post
page=urllib2.urlopen(req).read()
File "/usr/lib/python2.7/urllib2.py", line 127, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.7/urllib2.py", line 407, in open
response = meth(req, response)
File "/usr/lib/python2.7/urllib2.py", line 520, in http_response
'http', request, response, code, msg, hdrs)
File "/usr/lib/python2.7/urllib2.py", line 439, in error
result = self._call_chain(*args)
File "/usr/lib/python2.7/urllib2.py", line 379, in _call_chain
result = func(*args)
File "/usr/lib/python2.7/urllib2.py", line 626, in http_error_302
return self.parent.open(new, timeout=req.timeout)
File "/usr/lib/python2.7/urllib2.py", line 407, in open
response = meth(req, response)
File "/usr/lib/python2.7/urllib2.py", line 520, in http_response
'http', request, response, code, msg, hdrs)
File "/usr/lib/python2.7/urllib2.py", line 445, in error
return self._call_chain(*args)
File "/usr/lib/python2.7/urllib2.py", line 379, in _call_chain
result = func(*args)
File "/usr/lib/python2.7/urllib2.py", line 528, in http_error_default
raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
urllib2.HTTPError: HTTP Error 500: Internal Server Error
save your time, use this requests lib for httpRequests,
simple app
import requests
url = 'http://url.com'
query = {'field': value}
res = requests.post(url, data=query)
print(res.text)
Related
I'm using urllib.request.
I'm trying to use the BingX Swap Api with Python and I'm getting this error:
I'm trying to use the BingX Swap Api with Python and I'm getting this error:
I'm trying to use the BingX Swap Api with Python and I'm getting this error:
I'm trying to use the BingX Swap Api with Python and I'm getting this error:
Traceback (most recent call last):
File "Desktop\MyBot\MyApi.py", line 118, in <module>
main()
File "Desktop\MyBot\MyApi.py", line 103, in main
print("getLatestPrice:", getLatestPrice())
^^^^^^^^^^^^^^^^
File "Desktop\MyBot\MyApi.py", line 70, in getLatestPrice
return post(url, paramsStr)
^^^^^^^^^^^^^^^^^^^^
File "Desktop\MyBot\MyApi.py", line 19, in post
return urllib.request.urlopen(req).read()
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Python\Python311\Lib\urllib\request.py", line 216, in urlopen
return opener.open(url, data, timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Python\Python311\Lib\urllib\request.py", line 525, in open
response = meth(req, response)
^^^^^^^^^^^^^^^^^^^
File "Python\Python311\Lib\urllib\request.py", line 634, in http_response
response = self.parent.error(
^^^^^^^^^^^^^^^^^^
File "Python\Python311\Lib\urllib\request.py", line 563, in error
return self._call_chain(*args)
^^^^^^^^^^^^^^^^^^^^^^^
File "Python\Python311\Lib\urllib\request.py", line 496, in _call_chain
result = func(*args)
^^^^^^^^^^^
File "Python\Python311\Lib\urllib\request.py", line 643, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 405: Method Not Allowed
I don't know what I'm doing wrong, I think that my code is ok.
This is My Code:
import urllib.request
import json
import base64
import hmac
import time
APIURL = "https://api-swap-rest.bingbon.pro"
APIKEY = "xxxxxxxxxxxxx"
SECRETKEY = "xxxxxxxxxxxxx"
def genSignature(path, method, paramsMap):
sortedKeys = sorted(paramsMap)
paramsStr = "&".join(["%s=%s" % (x, paramsMap[x]) for x in sortedKeys])
paramsStr = method + path + paramsStr
return hmac.new(SECRETKEY.encode("utf-8"), paramsStr.encode("utf-8"), digestmod="sha256").digest()
def post(url, body):
req = urllib.request.Request(url, data=body.encode("utf-8"), headers={'User-Agent': 'Mozilla/5.0'})
return urllib.request.urlopen(req).read()
def getLatestPrice():
paramsMap = {
"symbol": "BTC-USDT",
}
sortedKeys = sorted(paramsMap)
paramsStr = "&".join(["%s=%s" % (x, paramsMap[x]) for x in sortedKeys])
paramsStr += "&sign=" + urllib.parse.quote(base64.b64encode(genSignature("/api/v1/market/getLatestPrice", "GET", paramsMap)))
url = "%s/api/v1/market/getLatestPrice" % APIURL
return post(url, paramsStr)
def main():
print("getLatestPrice:", getLatestPrice())
if __name__ == "__main__":
main()
Closed. This question needs debugging details. It is not currently accepting answers.
Edit the question to include desired behavior, a specific problem or error, and the shortest code necessary to reproduce the problem. This will help others answer the question.
Closed 1 year ago.
Improve this question
I have error I want help for it please help me
the error ;
Traceback (most recent call last):
File "c:/porgrammer/pythons/New folder/Main.py", line 6, in <module>
yt = YouTube(url)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pytube\__main__.py", line 91, in __init__
self.prefetch()
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pytube\__main__.py", line 181, in prefetch
self.vid_info_raw = request.get(self.vid_info_url)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pytube\request.py", line 36, in get
return _execute_request(url).read().decode("utf-8")
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pytube\request.py", line 24, in _execute_request
return urlopen(request) # nosec
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 532, in open
response = meth(req, response)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 642, in http_response
'http', request, response, code, msg, hdrs)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 564, in error
result = self._call_chain(*args)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 504, in _call_chain
result = func(*args)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 756, in http_error_302
return self.parent.open(new, timeout=req.timeout)
response = meth(req, response)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 642, in http_response
'http', request, response, code, msg, hdrs)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 570, in error
return self._call_chain(*args)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 504, in _call_chain
result = func(*args)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 650, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 404: Not Found
PS C:\porgrammer\pythons\New folder> & C:/Users/belba/AppData/Local/Programs/Python/Python36-32/python.exe "c:/porgrammer/pythons/New folder/Main.py"
Enter The Link Of The Video: https://www.youtube.com/watch?v=UoHpvfgj3WA
The Link Is : https://www.youtube.com/watch?v=UoHpvfgj3WA
Traceback (most recent call last):
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 1318, in do_open
encode_chunked=req.has_header('Transfer-encoding'))
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1026, in _send_output
self.send(msg)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 964, in send
self.connect()
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1392, in connect
super().connect()
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 936, in connect
(self.host,self.port), self.timeout, self.source_address)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\socket.py", line 704, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\socket.py", line 745, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno 11001] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "c:/porgrammer/pythons/New folder/Main.py", line 6, in <module>
yt = YouTube(url)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pytube\__main__.py", line 91, in __init__
self.prefetch()
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pytube\__main__.py", line 162, in prefetch
self.watch_html = request.get(url=self.watch_url)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pytube\request.py", line 36, in get
return _execute_request(url).read().decode("utf-8")
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pytube\request.py", line 24, in _execute_request
return urlopen(request) # nosec
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 526, in open
response = self._open(req, data)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 544, in _open
'_open', req)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 504, in _call_chain
result = func(*args)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 1361, in https_open
context=self._context, check_hostname=self._check_hostname)
File "C:\Users\username\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 1320, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 11001] getaddrinfo failed>
The Code ;
from pytube import YouTube
url = input("Enter The Link Of The Video: ")
print("The Link Is : " + url)
print("Copyright Srj")
yt = YouTube(url)
title = yt.title * 60
print("The Title Of The Video Is : " + title )
print("The Views Of The Video Is :" + yt.views)
print("The Lenght Of The Video Is : " + yt.length)
print("The Channel Who Post The Video Is : " + yt.rating)
print("The Description Of The Video Is : " + yt.description)
print("Copyright Srj")
YN = input("Are You Sure You Want To Install The Video?")
if YN == "yes" :
ys = yt.streams.get_highest_resolution()
print("Downloading.......")
ys.download("C:\YT_DOWNLOADS")
print("Downlaod Completed!!")
print("Copyright Srj")
elif YN == "no" :
print("No Proplem")
print("Copyright Srj")
else :
print("Please Retry Again Because It Is Not yes Or no")
print("Copyright Srj")
I modified some things in the code but never got the saame error as you.
This code works for me and even with the exact link of your video :
from pytube import YouTube
url = input("Enter The Link Of The Video: ")
print("The Link Is : " + url)
print("Copyright Srj")
yt = YouTube(url)
title = yt.title
print("The Title Of The Video Is : " + title )
print(f"The Views Of The Video Is : {yt.views}")
print(f"The Lenght Of The Video Is : {yt.length}")
print(f"The Channel Who Post The Video Is : {yt.rating}")
print("The Description Of The Video Is : " + yt.description)
print("Copyright Srj")
while True:
YN = str(input("Are You Sure You Want To Install The Video ? (yes / no)\n"))
if YN.lower() == "yes" :
ys = yt.streams.get_highest_resolution()
print("Downloading.......")
ys.download("C:\YT_DOWNLOADS")
print("Downlaod Completed!!")
print("Copyright Srj")
break
elif YN.lower() == "no" :
print("No Proplem")
print("Copyright Srj")
break
else :
print("Please Retry Again Because It Is Not yes Or no")
print("Copyright Srj")
Hope this works for you :)
So i was trying to follow this tutorial https://www.youtube.com/watch?v=Lg4T9iJkwhE&t=155s to achieve image detection with YOLO.
and when trying to run this code to auto download images of cell phones but it didn't work
import os
import urllib.request as ulib
from bs4 import BeautifulSoup as Soup
import json
url_a = 'https://www.google.com/search?ei=1m7NWePfFYaGmQG51q7IBg&hl=en&q={}'
url_b = '\&tbm=isch&ved=0ahUKEwjjovnD7sjWAhUGQyYKHTmrC2kQuT0I7gEoAQ&start={}'
url_c = '\&yv=2&vet=10ahUKEwjjovnD7sjWAhUGQyYKHTmrC2kQuT0I7gEoAQ.1m7NWePfFYaGmQG51q7IBg'
url_d = '\.i&ijn=1&asearch=ichunk&async=_id:rg_s,_pms:s'
url_base = ''.join((url_a, url_b, url_c, url_d))
headers = {'User-Agent': 'Chrome/41.0.2228.0 Safari/537.36'}
def get_links(search_name):
search_name = search_name.replace(' ', '+')
url = url_base.format(search_name, 0)
request = ulib.Request(url, None, headers)
json_string = ulib.urlopen(request).read()
page = json.loads(json_string)
new_soup = Soup(page[1][1], 'lxml')
images = new_soup.find_all('img')
links = [image['src'] for image in images]
return links
def save_images(links, search_name):
directory = search_name.replace(' ', '_')
if not os.path.isdir(directory):
os.mkdir(directory)
for i, link in enumerate(links):
savepath = os.path.join(directory, '{:06}.png'.format(i))
ulib.urlretrieve(link, savepath)
if __name__ == '__main__':
search_name = 'cell phones'
links = get_links(search_name)
save_images(links, search_name)
I got bunch of errors like this :
C:\Python36\python.exe "C:/dark/darkflow-master/new_model_data/part5 - get_images.py"
Traceback (most recent call last):
File "C:/dark/darkflow-master/new_model_data/part5 - get_images.py", line 39, in <module>
links = get_links(search_name)
File "C:/dark/darkflow-master/new_model_data/part5 - get_images.py", line 19, in get_links
json_string = ulib.urlopen(request).read()
File "C:\Python36\lib\urllib\request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "C:\Python36\lib\urllib\request.py", line 532, in open
response = meth(req, response)
File "C:\Python36\lib\urllib\request.py", line 642, in http_response
'http', request, response, code, msg, hdrs)
File "C:\Python36\lib\urllib\request.py", line 570, in error
return self._call_chain(*args)
File "C:\Python36\lib\urllib\request.py", line 504, in _call_chain
result = func(*args)
File "C:\Python36\lib\urllib\request.py", line 650, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 400: Bad Request
Process finished with exit code 1
Someone please help me fix this mess,thanks.
So I have roughly 6600 image URLs that I want to read them from a pickle file in order to download their images locally. The two main problems I could not solve:
1) URLs that throw UnicodeEncodeError error. I tried many available solution online and none worked so I ended up forgetting about that URL but I prefer if there is a way to hack it. This is URLs that contain Chinese of Polaska characters for example.
2) The other error is as follows (one example):
ssl.CertificateError: hostname 'www.straitstimes.com.sg' doesn't match either of 'gp1.adn.edgecastcdn.net', 'gs1.adn.edgecastcdn.net', 'ne1.adn.edgecastcdn.net', 'www.uship.com', 'www.bluefly.com', 'www.belleandclive.com', 'is.belleandclive.com', 'connections.cochlear.com', 'assets.pokemon.com', 'www.shopperschoice.com', 'www.biznessapps.com', 'cdn.shocho.co', 'secure.hibustudio.com', 'www.stardoll.com', 'adn.wiredrive.com', 'www.speedtest.net', 'www.cduniverse.com', 'ak-site-origin-cover.cduniverse.com', 'cover.cduniverse.com', 'g.cduniverse.com', 'www.renttherunway.com', 'cdn2.navexglobal.com', 'www.chdist.com', 'www.thefanorama.com', 'cdn2.mediasilo.com', 'cdn.citadoncw.com', 'www.woodcraft.com', 'marketing-admin.upsight-api.com', 'www.edgecast.com'
It happened when I needed to download the image from https://static.straitstimes.com.sg/sites/default/files/styles/x_large/public/articles/2017/09/23/ST_20170923_WONZ_3440048.jpg?itok=-iG-zSvr as you see the image is available.
I wonder how my code below could be modified to overcome these two issues rather than try except phrase.
import pickle
import urllib.request
import re
import requests
from urllib.parse import quote
# img = urllib.request.urlopen(quote(value))
# img = urllib.urlopen(quote(value))
import time
count = 0
with open('images.pickle', 'rb') as handle:
b = pickle.load(handle)
print(len(b))
for key, value in b.items():
print(value)
print(key, value)
#value = iriToUri(value)
if value != 'NA':
count += 1
print(value)
try:
img = urllib.request.urlopen(quote(value, "\./_-:"))
#img = requests.get(value)
split = urllib.parse.urlsplit(value)
extension = split.path.split(".")[-1]
print(extension)
if extension.lower() == 'jpg':
filename = "immigration_images/" + str(key) + ".jpg"
elif extension.lower() == 'jpeg':
filename = "immigration_images/" + str(key) + ".jpeg"
elif extension.lower() == 'ico':
filename = "immigration_images/" + str(key) + ".ico"
else:
filename = "immigration_images/" + str(key) + ".png"
img_extension = img.info()['Content-Type']
#print(img_extension)
if img_extension:
if 'jpeg' in img_extension:
filename = "immigration_images/" + str(key) + ".jpeg"
elif 'jpg' in img_extension:
filename = "immigration_images/" + str(key) + '.jpg'
elif 'png' in img_extension:
filename = "immigration_images/" + str(key) + '.png'
#urllib.request.urlretrieve(value, filename)
urllib.request.urlretrieve(value, filename)
except (urllib.error.ContentTooShortError, urllib.error.URLError, urllib.error.HTTPError, UnicodeEncodeError) as e:
print(e)
continue
print(count)
I am using Python 3.6.3 :: Anaconda custom (64-bit) in CentOS Linux release 7.4.1708 (Core)
trackback:
Traceback (most recent call last):
File "/scratch2/news_bias/test_pickle.py", line 39, in <module>
img = urllib.request.urlopen(quote(value, "\./_-:"))
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 532, in open
response = meth(req, response)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 642, in http_response
'http', request, response, code, msg, hdrs)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 564, in error
result = self._call_chain(*args)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 504, in _call_chain
result = func(*args)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 756, in http_error_302
return self.parent.open(new, timeout=req.timeout)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 526, in open
response = self._open(req, data)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 544, in _open
'_open', req)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 504, in _call_chain
result = func(*args)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 1361, in https_open
context=self._context, check_hostname=self._check_hostname)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 1318, in do_open
encode_chunked=req.has_header('Transfer-encoding'))
File "/scratch/sjn/anaconda/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/scratch/sjn/anaconda/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/scratch/sjn/anaconda/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/scratch/sjn/anaconda/lib/python3.6/http/client.py", line 1026, in _send_output
self.send(msg)
File "/scratch/sjn/anaconda/lib/python3.6/http/client.py", line 964, in send
self.connect()
File "/scratch/sjn/anaconda/lib/python3.6/http/client.py", line 1400, in connect
server_hostname=server_hostname)
File "/scratch/sjn/anaconda/lib/python3.6/ssl.py", line 407, in wrap_socket
_context=self, _session=session)
File "/scratch/sjn/anaconda/lib/python3.6/ssl.py", line 814, in __init__
self.do_handshake()
File "/scratch/sjn/anaconda/lib/python3.6/ssl.py", line 1068, in do_handshake
self._sslobj.do_handshake()
File "/scratch/sjn/anaconda/lib/python3.6/ssl.py", line 694, in do_handshake
match_hostname(self.getpeercert(), self.server_hostname)
File "/scratch/sjn/anaconda/lib/python3.6/ssl.py", line 327, in match_hostname
% (hostname, ', '.join(map(repr, dnsnames))))
I am running a python script with Phontomjs and Selenium. I am facing timeout issue. It is stopping after 20-50min. I need a solution so that I can run my script without this timeout issue. where is the problem please and how can I solve it?
The input file cannot be read or no in proper format.
Traceback (most recent call last):
File "links_crawler.py", line 147, in <module>
crawler.Run()
File "links_crawler.py", line 71, in Run
self.checkForNextPages()
File "links_crawler.py", line 104, in checkForNextPages
self.next.click()
File "/home/dev/.local/lib/python2.7/site-packages/selenium/webdriver/remote/webelement.py", line 75, in click
self._execute(Command.CLICK_ELEMENT)
File "/home/dev/.local/lib/python2.7/site-packages/selenium/webdriver/remote/webelement.py", line 454, in _execute
return self._parent.execute(command, params)
File "/home/dev/.local/lib/python2.7/site-packages/selenium/webdriver/remote/webdriver.py", line 199, in execute
response = self.command_executor.execute(driver_command, params)
File "/home/dev/.local/lib/python2.7/site-packages/selenium/webdriver/remote/remote_connection.py", line 395, in execute
return self._request(command_info[0], url, body=data)
File "/home/dev/.local/lib/python2.7/site-packages/selenium/webdriver/remote/remote_connection.py", line 463, in _request
resp = opener.open(request, timeout=self._timeout)
File "/usr/lib/python2.7/urllib2.py", line 431, in open
response = self._open(req, data)
File "/usr/lib/python2.7/urllib2.py", line 449, in _open
'_open', req)
File "/usr/lib/python2.7/urllib2.py", line 409, in _call_chain
result = func(*args)
File "/usr/lib/python2.7/urllib2.py", line 1227, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/usr/lib/python2.7/urllib2.py", line 1200, in do_open
r = h.getresponse(buffering=True)
File "/usr/lib/python2.7/httplib.py", line 1127, in getresponse
response.begin()
File "/usr/lib/python2.7/httplib.py", line 453, in begin
version, status, reason = self._read_status()
File "/usr/lib/python2.7/httplib.py", line 417, in _read_status
raise BadStatusLine(line)
httplib.BadStatusLine: ''
Code:
class Crawler():
def __init__(self,where_to_save, verbose = 0):
self.link_to_explore = ''
self.TAG_RE = re.compile(r'<[^>]+>')
self.TAG_SCRIPT = re.compile(r'<(script).*?</\1>(?s)')
if verbose == 1:
self.driver = webdriver.Firefox()
else:
self.driver = webdriver.PhantomJS()
self.links = []
self.next = True
self.where_to_save = where_to_save
self.logs = self.where_to_save + "/logs"
self.outputs = self.where_to_save + "/outputs"
self.logfile = ''
self.rnd = 0
try:
os.stat(self.logs)
except:
os.makedirs(self.logs)
try:
os.stat(self.outputs)
except:
os.makedirs(self.outputs)
try:
fin = open(file_to_read,"r")
FileContent = fin.read()
fin.close()
crawler =Crawler(where_to_save)
data = FileContent.split("\n")
for info in data:
if info!="":
to_process = info.split("|")
link = to_process[0].strip()
category = to_process[1].strip().replace(' ','_')
print "Processing the link: " + link : " + info
crawler.Init(link,category)
crawler.Run()
crawler.End()
crawler.closeSpider()
except:
print "The input file cannot be read or no in proper format."
raise
If you don't want Timeout to stop your script you can catch the exception
selenium.common.exceptions.TimeoutException and pass it.
You can set the default page load timeout using the set_page_load_timeout() method of webdriver.
Like this
driver.set_page_load_timeout(10)
This will throw a TimeoutException if your page didn't load in 10 seconds.
EDIT:
Forgot to mention that you will have to put your code in a loop.
Add import
from selenium.common.exceptions import TimeoutException
while True:
try:
# Your code here
break # Loop will exit
except TimeoutException:
pass