Related
I've made a very basic share price prediction script which essentially uses linear regression. Not accurate, but it's proof of concept for a uni project.
I want to add these prices to my website using the new PyScript Library. However, when i run the code on my website, I get this error:
PythonError: Traceback (most recent call last): File "/lib/python3.10/asyncio/futures.py", line 201, in result raise self._exception File "/lib/python3.10/asyncio/tasks.py", line 232, in __step result = coro.send(None) File "/lib/python3.10/site-packages/_pyodide/_base.py", line 506, in eval_code_async await CodeRunner( File "/lib/python3.10/site-packages/_pyodide/_base.py", line 357, in run_async coroutine = eval(self.code, globals, locals) File "", line 6, in File "/lib/python3.10/site-packages/pandas/util/_decorators.py", line 311, in wrapper return func(*args, **kwargs) File "/lib/python3.10/site-packages/pandas/io/parsers/readers.py", line 680, in read_csv return _read(filepath_or_buffer, kwds) File "/lib/python3.10/site-packages/pandas/io/parsers/readers.py", line 575, in _read parser = TextFileReader(filepath_or_buffer, **kwds) File "/lib/python3.10/site-packages/pandas/io/parsers/readers.py", line 933, in init self._engine = self._make_engine(f, self.engine) File "/lib/python3.10/site-packages/pandas/io/parsers/readers.py", line 1217, in _make_engine self.handles = get_handle( # type: ignore[call-overload] File "/lib/python3.10/site-packages/pandas/io/common.py", line 670, in get_handle ioargs = _get_filepath_or_buffer( File "/lib/python3.10/site-packages/pandas/io/common.py", line 339, in _get_filepath_or_buffer with urlopen(req_info) as req: File "/lib/python3.10/site-packages/pandas/io/common.py", line 239, in urlopen return urllib.request.urlopen(*args, **kwargs) File "/lib/python3.10/urllib/request.py", line 216, in urlopen return opener.open(url, data, timeout) File "/lib/python3.10/urllib/request.py", line 519, in open response = self._open(req, data) File "/lib/python3.10/urllib/request.py", line 541, in _open return self._call_chain(self.handle_open, 'unknown', File "/lib/python3.10/urllib/request.py", line 496, in _call_chain result = func(*args) File "/lib/python3.10/urllib/request.py", line 1419, in unknown_open raise URLError('unknown url type: %s' % type) urllib.error.URLError:
This solution worked absolutely fine in pyCharm, so im assunming it's something to do with PyScript?
I'll link the code below.
<py-script>
import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression
urlAAPL = "https://raw.githubusercontent.com/Dan168/-DanCreates/main/AAPL.csv"
dataAAPL = pd.read_csv(urlAAPL)
XAAPL = dataAAPL.iloc[:, 0].values.reshape(-1, 1)
YAAPL = dataAAPL.iloc[:, 1].values.reshape(-1, 1)
urlAMZN = "https://raw.githubusercontent.com/Dan168/-DanCreates/main/AMZN.csv"
dataAMZN = pd.read_csv(urlAMZN)
XAMZN = dataAMZN.iloc[:, 0].values.reshape(-1, 1)
YAMZN = dataAMZN.iloc[:, 1].values.reshape(-1, 1)
urlTSLA = "https://raw.githubusercontent.com/Dan168/-DanCreates/main/TSLA.csv"
dataTSLA = pd.read_csv(urlTSLA)
XTSLA = dataTSLA.iloc[:, 0].values.reshape(-1, 1)
YTSLA = dataTSLA.iloc[:, 1].values.reshape(-1, 1)
urlNIO = "https://raw.githubusercontent.com/Dan168/-DanCreates/main/NIO.csv"
dataNIO = pd.read_csv(urlNIO)
XNIO = dataNIO.iloc[:, 0].values.reshape(-1, 1)
YNIO = dataNIO.iloc[:, 1].values.reshape(-1, 1)
def Predict(X, Y, Name):
lr = LinearRegression()
lr.fit(X, Y)
Y_pred = lr.predict(X)
oneDay = 127
oneWeek = 134
oneMonth = 156
print(str(Name))
print("Price prediction one day day from now: ")
print(lr.predict(np.array([oneDay]).reshape(-1, 1)))
print("Price prediction one week day from now: ")
print(lr.predict(np.array([oneWeek]).reshape(-1, 1)))
print("Price prediction one month day from now: ")
print(lr.predict(np.array([oneMonth]).reshape(-1, 1)))
Predict(XAAPL, YAAPL, "AAPL")
Predict(XNIO, YNIO, "NIO")
Predict(XTSLA, YTSLA, "TSLA")
Predict(XAMZN, YAMZN, "AMZN")
</py-script>
Ran the code in PyCharm and it worked absolutely fine. Ran fine on my website before when it was just checking one CSV. When I added the method and more stocks to check, this is when I encounted the error - hence making me thing it is something to do with the method??
Thanks in advance
I am trying to generate a qr code from text, and then insert into a reportlab pdf.
My code:
def qr_code_as_image(text):
from io import BytesIO
print("In show_qr")
img = generate_qr_code(text)
print(img, type(img))
i = Image(img)
print(i, type(i))
return i
def add_patient_header_with_qr(self):
line1 = ("Name", self.linkedcustomer.name,
"Age", self.linkedcustomer.age())
line2 = ("MRD No.", self.linkedcustomer.cstid,
"Date", self.prescription_time)
line3 = ("No.", "#", "Doctor", self.doc.name)
datatb = [line1, line2, line3]
patientdetailstable = Table(datatb)
patientdetailstable.setStyle(self.patientdetails_style)
col1 = patientdetailstable
checkin_url = reverse('clinicemr', args=[self.checkin.checkinno])
qr_image = qr_code_as_image(checkin_url)
qr_image.hAlign = 'LEFT'
col2 = Table([[qr_image]])
tblrow1 = Table([[col1, col2]], colWidths=None)
tblrow1.setStyle(self.table_left_top_align)
self.elements.append(tblrow1)
def final_generate(self, footer_content, action=None):
with NamedTemporaryFile(mode='w+b') as temp:
from django.http import FileResponse, Http404
from functools import partial
# use the temp file
cmd = "cat " + str(temp.name)
print(os.system(cmd))
print(footer_content, type(footer_content))
doc = SimpleDocTemplate(
temp.name,
pagesize=A4,
rightMargin=20,
leftMargin=20,
topMargin=20,
bottomMargin=80,
allowSplitting=1,
title="Prescription",
author="System.com")
frame = Frame(doc.leftMargin, doc.bottomMargin, doc.width, doc.height,
id='normal')
template = PageTemplate(
id='test',
frames=frame,
onPage=partial(footer, content=footer_content)
)
doc.addPageTemplates([template])
doc.build(self.elements,
onFirstPage=partial(footer, content=footer_content),
onLaterPages=partial(footer, content=footer_content)
)
print(f'Generated {temp.name}')
I get the following output:
2020-11-29 13: 06: 33, 915 django.request ERROR Internal Server Error: / clinic/presc/k-0NGpApcg
Traceback(most recent call last):
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/lib/utils.py", line 655, in open_for_read
return open_for_read_by_name(name, mode)
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/lib/utils.py", line 599, in open_for_read_by_name
return open(name, mode)
ValueError: embedded null byte
During handling of the above exception, another exception occurred:
Traceback(most recent call last):
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/lib/utils.py", line 658, in open_for_read
return getBytesIO(datareader(name) if name[:5].lower() == 'data:' else urlopen(name).read())
File "/usr/lib/python3.6/urllib/request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "/usr/lib/python3.6/urllib/request.py", line 517, in open
req.timeout = timeout
AttributeError: 'bytes' object has no attribute 'timeout'
During handling of the above exception, another exception occurred:
Traceback(most recent call last):
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/django/core/handlers/exception.py", line 34, in inner
response = get_response(request)
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/django/core/handlers/base.py", line 115, in _get_response
response = self.process_exception_by_middleware(e, request)
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/django/core/handlers/base.py", line 113, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/home/joel/myappointments/clinic/views.py", line 6879, in GoGetPrescription
clinicobj = clinicobj,
File "/home/joel/myappointments/clinic/views.py", line 16222, in PDFPrescriptions
return prescription.generate_pdf(action=action, rating=True)
File "/home/joel/myappointments/clinic/views.py", line 15415, in generate_pdf
return self.final_generate(footer_content, action=action)
File "/home/joel/myappointments/clinic/views.py", line 15447, in final_generate
onLaterPages = partial(footer, content=footer_content)
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/platypus/doctemplate.py", line 1291, in build
BaseDocTemplate.build(self, flowables, canvasmaker=canvasmaker)
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/platypus/doctemplate.py", line 1056, in build
self.handle_flowable(flowables)
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/platypus/doctemplate.py", line 912, in handle_flowable
if frame.add(f, canv, trySplit=self.allowSplitting):
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/platypus/frames.py", line 174, in _add
w, h = flowable.wrap(aW, h)
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/platypus/tables.py", line 1206, in wrap
self._calc(availWidth, availHeight)
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/platypus/tables.py", line 641, in _calc
W = self._calcPreliminaryWidths(availWidth) # widths
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/platypus/tables.py", line 754, in _calcPreliminaryWidths
new = elementWidth(value, style) or 0
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/platypus/tables.py", line 518, in _elementWidth
w = v.minWidth() # should be all flowables
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/platypus/tables.py", line 873, in minWidth
style.leftPadding+style.rightPadding)
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/platypus/tables.py", line 512, in _elementWidth
if hasattr(v, 'drawWidth') and isinstance(v.drawWidth, (int, float)): return v.drawWidth
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/platypus/flowables.py", line 494, in __getattr__
self._setup_inner()
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/platypus/flowables.py", line 455, in _setup_inner
img=self._img
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/platypus/flowables.py", line 488, in __getattr__
self._img=ImageReader(self._file)
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/lib/utils.py", line 813, in __init__
annotateException('\nfileName=%r identity=%s' %
(fileName, self.identity()))
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/lib/utils.py", line 1394, in annotateException
rl_reraise(t, v, b)
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/lib/utils.py", line 147, in rl_reraise
raise v
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/lib/utils.py", line 777, in __init__
self.fp=open_for_read(fileName, 'b')
File "/home/joel/myappointments/venv/lib/python3.6/site-packages/reportlab/lib/utils.py", line 660, in open_for_read
raise IOError('Cannot open resource "%s"' % name)
OSError: Cannot open resource "b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\xd2\x00\x00\x00\xd2\x01\x00\x00\x00\x00\x17\xe2\xa3\xef\x00\x00\x01$IDATx\x9c\xed\x98An\xc4 \x10\x04k\x16\xdf\xf1\x8f\xe0gyS~\x80\x9f\x92\x1f\xe0;\xab\xde\x03\xc6\xeb\x1c"\xe5d\xd0\xda\x1c\xd0 #\xcbRk4\x9e\xee\x01\xb6\xe5$\xa9 \xe5v\xc3\x83\xbf\xd7\xe7cA\x92\x94\xc1"N\x16k\x86\xa4\xd1x\x9e\x8bA\xc8#\xc8NJ\xbe e\'\xc0]</\x07\xccb\xdb\xfas\xe9\x8eM\xefP\xac\x13b\xed\xc6e0\xccKJ\x80\xd9\xecd\x11\x90T\xfap\x19\x06\xdb\x97\x13!;\xd5v\xd3\x87\xcbH\xd8\xa4=\x14\xeb\xd3\xc0\xb7\x9be$\x9e\x9d\xea\xa5V\x89/u\xab\xca\x94F\xe2yz^\x94\xbc\x04^\xda\x8ePe{,\x9e}\xeaE\xe9\xed\xe6\xe0\xae\x17\xa0\xa6#\xf9\xb2\x9b;\xe9\x1f\xdf}:\xc6A\x80v=\xbau\xba\xd5\xcb\xef_hk7#\xf1\xec\xee_\xf0\x92\x94\x9d.\xde_\xda<\xdd\xde\x19R\xb5\xbfW\xcf\xcb\x03V3\x8b\xb4\x911\xfc\x98\xd9\xd7\xe5\xfb\xcb\x11[f\'\x96\x19\x80\xa7\x8d\xcb\xf3\x0c\xec0O\x13\xbe\xa7b\xf8\x0c\x8b\xdd\xben\xef/\xc0\xf68\xb5E#\xf1\xec\xaaGU\xac\x9d\x08\xba\xba\xdf}\x01<\xf7\xbf\x8cN\xed-\x8a\x00\x00\x00\x00IEND\xaeB`\x82'"
fileName=b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\xd2\x00\x00\x00\xd2\x01\x00\x00\x00\x00\x17\xe2\xa3\xef\x00\x00\x01$IDATx\x9c\xed\x98An\xc4 \x10\x04k\x16\xdf\xf1\x8f\xe0gyS~\x80\x9f\x92\x1f\xe0;\xab\xde\x03\xc6\xeb\x1c"\xe5d\xd0\xda\x1c\xd0#\xcbRk4\x9e\xee\x01\xb6\xe5$\xa9 \xe5v\xc3\x83\xbf\xd7\xe7cA\x92\x94\xc1"N\x16k\x86\xa4\xd1x\x9e\x8bA\xc8#\xc8NJ\xbe e\'\xc0]</\x07\xccb\xdb\xfas\xe9\x8eM\xefP\xac\x13b\xed\xc6e0\xccKJ\x80\xd9\xecd\x11\x90T\xfap\x19\x06\xdb\x97\x13!;\xd5v\xd3\x87\xcbH\xd8\xa4=\x14\xeb\xd3\xc0\xb7\x9be$\x9e\x9d\xea\xa5V\x89/u\xab\xca\x94F\xe2yz^\x94\xbc\x04^\xda\x8ePe{,\x9e}\xeaE\xe9\xed\xe6\xe0\xae\x17\xa0\xa6#\xf9\xb2\x9b;\xe9\x1f\xdf}:\xc6A\x80v=\xbau\xba\xd5\xcb\xef_hk7#\xf1\xec\xee_\xf0\x92\x94\x9d.\xde_\xda<\xdd\xde\x19R\xb5\xbfW\xcf\xcb\x03V3\x8b\xb4\x911\xfc\x98\xd9\xd7\xe5\xfb\xcb\x11[f\'\x96\x19\x80\xa7\x8d\xcb\xf3\x0c\xec0O\x13\xbe\xa7b\xf8\x0c\x8b\xdd\xben\xef/\xc0\xf68\xb5E#\xf1\xec\xaaGU\xac\x9d\x08\xba\xba\xdf}\x01<\xf7\xbf\x8cN\xed-\x8a\x00\x00\x00\x00IEND\xaeB`\x82' identity=[ImageReader#0x7f1e0987ecf8 filename=b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\xd2\x00\x00\x00\xd2\x01\x00\x00\x00\x00\x17\xe2\xa3\xef\x00\x00\x01$IDATx\x9c\xed\x98An\xc4 \x10\x04k\x16\xdf\xf1\x8f\xe0gyS~\x80\x9f\x92\x1f\xe0;\xab\xde\x03\xc6\xeb\x1c"\xe5d\xd0\xda\x1c\xd0#\xcbRk4\x9e\xee\x01\xb6\xe5$\xa9 \xe5v\xc3\x83\xbf\xd7\xe7cA\x92\x94\xc1"N\x16k\x86\xa4\xd1x\x9e\x8bA\xc8#\xc8NJ\xbe e\'\xc0]</\x07\xccb\xdb\xfas\xe9\x8eM\xefP\xac\x13b\xed\xc6e0\xccKJ\x80\xd9\xecd\x11\x90T\xfap\x19\x06\xdb\x97\x13!;\xd5v\xd3\x87\xcbH\xd8\xa4=\x14\xeb\xd3\xc0\xb7\x9be$\x9e\x9d\xea\xa5V\x89/u\xab\xca\x94F\xe2yz^\x94\xbc\x04^\xda\x8ePe{,\x9e}\xeaE\xe9\xed\xe6\xe0\xae\x17\xa0\xa6#\xf9\xb2\x9b;\xe9\x1f\xdf}:\xc6A\x80v=\xbau\xba\xd5\xcb\xef_hk7#\xf1\xec\xee_\xf0\x92\x94\x9d.\xde_\xda<\xdd\xde\x19R\xb5\xbfW\xcf\xcb\x03V3\x8b\xb4\x911\xfc\x98\xd9\xd7\xe5\xfb\xcb\x11[f\'\x96\x19\x80\xa7\x8d\xcb\xf3\x0c\xec0O\x13\xbe\xa7b\xf8\x0c\x8b\xdd\xben\xef/\xc0\xf68\xb5E#\xf1\xec\xaaGU\xac\x9d\x08\xba\xba\xdf}\x01<\xf7\xbf\x8cN\xed-\x8a\x00\x00\x00\x00IEND\xaeB`\x82']
From the error, it appears that it is erroring out on getting the name of the image file. But there is no file. The image is being generated from BytesIO.
Your generate_qr_code function, which you did not show us, is NOT returning a BytesIO object. It's returning the raw bytes of the PNG image. When you print(img, type(img)), it told you it was of type "bytes", right? That's a string of bytes, not a BytesIO object. If you wrap those bytes into a BytesIO object, then the reportlab Image constructor will be able to handle it.
So I have roughly 6600 image URLs that I want to read them from a pickle file in order to download their images locally. The two main problems I could not solve:
1) URLs that throw UnicodeEncodeError error. I tried many available solution online and none worked so I ended up forgetting about that URL but I prefer if there is a way to hack it. This is URLs that contain Chinese of Polaska characters for example.
2) The other error is as follows (one example):
ssl.CertificateError: hostname 'www.straitstimes.com.sg' doesn't match either of 'gp1.adn.edgecastcdn.net', 'gs1.adn.edgecastcdn.net', 'ne1.adn.edgecastcdn.net', 'www.uship.com', 'www.bluefly.com', 'www.belleandclive.com', 'is.belleandclive.com', 'connections.cochlear.com', 'assets.pokemon.com', 'www.shopperschoice.com', 'www.biznessapps.com', 'cdn.shocho.co', 'secure.hibustudio.com', 'www.stardoll.com', 'adn.wiredrive.com', 'www.speedtest.net', 'www.cduniverse.com', 'ak-site-origin-cover.cduniverse.com', 'cover.cduniverse.com', 'g.cduniverse.com', 'www.renttherunway.com', 'cdn2.navexglobal.com', 'www.chdist.com', 'www.thefanorama.com', 'cdn2.mediasilo.com', 'cdn.citadoncw.com', 'www.woodcraft.com', 'marketing-admin.upsight-api.com', 'www.edgecast.com'
It happened when I needed to download the image from https://static.straitstimes.com.sg/sites/default/files/styles/x_large/public/articles/2017/09/23/ST_20170923_WONZ_3440048.jpg?itok=-iG-zSvr as you see the image is available.
I wonder how my code below could be modified to overcome these two issues rather than try except phrase.
import pickle
import urllib.request
import re
import requests
from urllib.parse import quote
# img = urllib.request.urlopen(quote(value))
# img = urllib.urlopen(quote(value))
import time
count = 0
with open('images.pickle', 'rb') as handle:
b = pickle.load(handle)
print(len(b))
for key, value in b.items():
print(value)
print(key, value)
#value = iriToUri(value)
if value != 'NA':
count += 1
print(value)
try:
img = urllib.request.urlopen(quote(value, "\./_-:"))
#img = requests.get(value)
split = urllib.parse.urlsplit(value)
extension = split.path.split(".")[-1]
print(extension)
if extension.lower() == 'jpg':
filename = "immigration_images/" + str(key) + ".jpg"
elif extension.lower() == 'jpeg':
filename = "immigration_images/" + str(key) + ".jpeg"
elif extension.lower() == 'ico':
filename = "immigration_images/" + str(key) + ".ico"
else:
filename = "immigration_images/" + str(key) + ".png"
img_extension = img.info()['Content-Type']
#print(img_extension)
if img_extension:
if 'jpeg' in img_extension:
filename = "immigration_images/" + str(key) + ".jpeg"
elif 'jpg' in img_extension:
filename = "immigration_images/" + str(key) + '.jpg'
elif 'png' in img_extension:
filename = "immigration_images/" + str(key) + '.png'
#urllib.request.urlretrieve(value, filename)
urllib.request.urlretrieve(value, filename)
except (urllib.error.ContentTooShortError, urllib.error.URLError, urllib.error.HTTPError, UnicodeEncodeError) as e:
print(e)
continue
print(count)
I am using Python 3.6.3 :: Anaconda custom (64-bit) in CentOS Linux release 7.4.1708 (Core)
trackback:
Traceback (most recent call last):
File "/scratch2/news_bias/test_pickle.py", line 39, in <module>
img = urllib.request.urlopen(quote(value, "\./_-:"))
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 532, in open
response = meth(req, response)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 642, in http_response
'http', request, response, code, msg, hdrs)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 564, in error
result = self._call_chain(*args)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 504, in _call_chain
result = func(*args)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 756, in http_error_302
return self.parent.open(new, timeout=req.timeout)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 526, in open
response = self._open(req, data)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 544, in _open
'_open', req)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 504, in _call_chain
result = func(*args)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 1361, in https_open
context=self._context, check_hostname=self._check_hostname)
File "/scratch/sjn/anaconda/lib/python3.6/urllib/request.py", line 1318, in do_open
encode_chunked=req.has_header('Transfer-encoding'))
File "/scratch/sjn/anaconda/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/scratch/sjn/anaconda/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/scratch/sjn/anaconda/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/scratch/sjn/anaconda/lib/python3.6/http/client.py", line 1026, in _send_output
self.send(msg)
File "/scratch/sjn/anaconda/lib/python3.6/http/client.py", line 964, in send
self.connect()
File "/scratch/sjn/anaconda/lib/python3.6/http/client.py", line 1400, in connect
server_hostname=server_hostname)
File "/scratch/sjn/anaconda/lib/python3.6/ssl.py", line 407, in wrap_socket
_context=self, _session=session)
File "/scratch/sjn/anaconda/lib/python3.6/ssl.py", line 814, in __init__
self.do_handshake()
File "/scratch/sjn/anaconda/lib/python3.6/ssl.py", line 1068, in do_handshake
self._sslobj.do_handshake()
File "/scratch/sjn/anaconda/lib/python3.6/ssl.py", line 694, in do_handshake
match_hostname(self.getpeercert(), self.server_hostname)
File "/scratch/sjn/anaconda/lib/python3.6/ssl.py", line 327, in match_hostname
% (hostname, ', '.join(map(repr, dnsnames))))
I am running a python script with Phontomjs and Selenium. I am facing timeout issue. It is stopping after 20-50min. I need a solution so that I can run my script without this timeout issue. where is the problem please and how can I solve it?
The input file cannot be read or no in proper format.
Traceback (most recent call last):
File "links_crawler.py", line 147, in <module>
crawler.Run()
File "links_crawler.py", line 71, in Run
self.checkForNextPages()
File "links_crawler.py", line 104, in checkForNextPages
self.next.click()
File "/home/dev/.local/lib/python2.7/site-packages/selenium/webdriver/remote/webelement.py", line 75, in click
self._execute(Command.CLICK_ELEMENT)
File "/home/dev/.local/lib/python2.7/site-packages/selenium/webdriver/remote/webelement.py", line 454, in _execute
return self._parent.execute(command, params)
File "/home/dev/.local/lib/python2.7/site-packages/selenium/webdriver/remote/webdriver.py", line 199, in execute
response = self.command_executor.execute(driver_command, params)
File "/home/dev/.local/lib/python2.7/site-packages/selenium/webdriver/remote/remote_connection.py", line 395, in execute
return self._request(command_info[0], url, body=data)
File "/home/dev/.local/lib/python2.7/site-packages/selenium/webdriver/remote/remote_connection.py", line 463, in _request
resp = opener.open(request, timeout=self._timeout)
File "/usr/lib/python2.7/urllib2.py", line 431, in open
response = self._open(req, data)
File "/usr/lib/python2.7/urllib2.py", line 449, in _open
'_open', req)
File "/usr/lib/python2.7/urllib2.py", line 409, in _call_chain
result = func(*args)
File "/usr/lib/python2.7/urllib2.py", line 1227, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/usr/lib/python2.7/urllib2.py", line 1200, in do_open
r = h.getresponse(buffering=True)
File "/usr/lib/python2.7/httplib.py", line 1127, in getresponse
response.begin()
File "/usr/lib/python2.7/httplib.py", line 453, in begin
version, status, reason = self._read_status()
File "/usr/lib/python2.7/httplib.py", line 417, in _read_status
raise BadStatusLine(line)
httplib.BadStatusLine: ''
Code:
class Crawler():
def __init__(self,where_to_save, verbose = 0):
self.link_to_explore = ''
self.TAG_RE = re.compile(r'<[^>]+>')
self.TAG_SCRIPT = re.compile(r'<(script).*?</\1>(?s)')
if verbose == 1:
self.driver = webdriver.Firefox()
else:
self.driver = webdriver.PhantomJS()
self.links = []
self.next = True
self.where_to_save = where_to_save
self.logs = self.where_to_save + "/logs"
self.outputs = self.where_to_save + "/outputs"
self.logfile = ''
self.rnd = 0
try:
os.stat(self.logs)
except:
os.makedirs(self.logs)
try:
os.stat(self.outputs)
except:
os.makedirs(self.outputs)
try:
fin = open(file_to_read,"r")
FileContent = fin.read()
fin.close()
crawler =Crawler(where_to_save)
data = FileContent.split("\n")
for info in data:
if info!="":
to_process = info.split("|")
link = to_process[0].strip()
category = to_process[1].strip().replace(' ','_')
print "Processing the link: " + link : " + info
crawler.Init(link,category)
crawler.Run()
crawler.End()
crawler.closeSpider()
except:
print "The input file cannot be read or no in proper format."
raise
If you don't want Timeout to stop your script you can catch the exception
selenium.common.exceptions.TimeoutException and pass it.
You can set the default page load timeout using the set_page_load_timeout() method of webdriver.
Like this
driver.set_page_load_timeout(10)
This will throw a TimeoutException if your page didn't load in 10 seconds.
EDIT:
Forgot to mention that you will have to put your code in a loop.
Add import
from selenium.common.exceptions import TimeoutException
while True:
try:
# Your code here
break # Loop will exit
except TimeoutException:
pass
At work I have to access/work with the Channel Advisor API
Source:
I'm attempting to perform a simple OrderService
from suds.client import Client
wsdl_url = 'https://api.channeladvisor.com/ChannelAdvisorAPI/v6/OrderService.asmx?WSDL'
service_url = 'https://api.channeladvisor.com/ChannelAdvisorAPI/v6/OrderService.asmx'
headers = {'Content-Type': 'text/xml; charset=utf-8'}
client = Client(wsdl_url, location = service_url, headers=headers)
Here is error:
Traceback (most recent call last):
File "test_suds.py", line 9, in <module>
client = Client(wsdl_url, location = service_url, headers=headers)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/suds-0.4.1-py2.7.egg/suds/client.py", line 112, in __init__
self.wsdl = reader.open(url)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/suds-0.4.1-py2.7.egg/suds/reader.py", line 152, in open
d = self.fn(url, self.options)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/suds-0.4.1-py2.7.egg/suds/wsdl.py", line 136, in __init__
d = reader.open(url)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/suds-0.4.1-py2.7.egg/suds/reader.py", line 79, in open
d = self.download(url)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/suds-0.4.1-py2.7.egg/suds/reader.py", line 95, in download
fp = self.options.transport.open(Request(url))
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/suds-0.4.1-py2.7.egg/suds/transport/https.py", line 60, in open
return HttpTransport.open(self, request)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/suds-0.4.1-py2.7.egg/suds/transport/http.py", line 62, in open
return self.u2open(u2request)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/suds-0.4.1-py2.7.egg/suds/transport/http.py", line 118, in u2open
return url.open(u2request, timeout=tm)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 404, in open
response = self._open(req, data)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 422, in _open
'_open', req)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 382, in _call_chain
result = func(*args)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 1222, in https_open
return self.do_open(httplib.HTTPSConnection, req)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 1184, in do_open
raise URLError(err)
urllib2.URLError: <urlopen error [Errno 54] Connection reset by peer>