Hello everyone I am pretty new to programming so please forgive me any noob mistakes.
I am trying to use urllib in python 3.6 to find a string on a webpage and the solution I found here is working great for most webpages i tested, for example:
from urllib.request import urlopen
>>>string = 'google'
>>>url = 'https://google.com/'
>>>webp=urlopen(url).read()
>>>isfound = webp.find(string.encode('utf-8'))
>>>print(isfound)
186
But when I try using the URL https://chan.sankakucomplex.com/ (NSFW) I get tons of errors and I don't know why.
I allready tried changing my DNS to 8.8.8.8 but this didn't change anything.
error:
Traceback (most recent call last):
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 1318, in do_open
encode_chunked=req.has_header('Transfer-encoding'))
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 1026, in _send_output
self.send(msg)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 964, in send
self.connect()
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 1392, in connect
super().connect()
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 936, in connect
(self.host,self.port), self.timeout, self.source_address)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\socket.py", line 704, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\socket.py", line 743, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno 11004] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:/Users/-/Desktop/python/searcher/stackex.py", line 5, in <module>
webp=urlopen(url).read()
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 526, in open
response = self._open(req, data)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 544, in _open
'_open', req)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 504, in _call_chain
result = func(*args)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 1361, in https_open
context=self._context, check_hostname=self._check_hostname)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 1320, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 11004] getaddrinfo failed>
urllib.error.URLError: <urlopen error [Errno 11004] getaddrinfo failed>
This tells me that your DNS server is refusing to translate the text address (chan.sankakucomplex.com) into an IP address. It's not an issue with Python.
Can you get to that site from a web browser running on the same machine where you're running your python code?
Related
today I'm try to use yahoo-finance library than some issue occur by code.
i provide code and occurred error.
here is library link https://pypi.org/project/yahoo-finance/
code:-
from yahoo_finance import Share
yahoo = Share('YHOO')
print(yahoo.get_open())
Error is
Traceback (most recent call last):
File "/usr/lib/python3.6/urllib/request.py", line 1318, in do_open
encode_chunked=req.has_header('Transfer-encoding'))
File "/usr/lib/python3.6/http/client.py", line 1254, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1300, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1249, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1036, in _send_output
self.send(msg)
File "/usr/lib/python3.6/http/client.py", line 974, in send
self.connect()
File "/usr/lib/python3.6/http/client.py", line 1407, in connect
super().connect()
File "/usr/lib/python3.6/http/client.py", line 946, in connect
(self.host,self.port), self.timeout, self.source_address)
File "/usr/lib/python3.6/socket.py", line 704, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
File "/usr/lib/python3.6/socket.py", line 745, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "main.py", line 3, in <module>
yahoo = Share('YHOO')
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/__init__.py", line 178, in __init__
self.refresh()
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/__init__.py", line 142, in refresh
self.data_set = self._fetch()
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/__init__.py", line 181, in _fetch
data = super(Share, self)._fetch()
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/__init__.py", line 134, in _fetch
data = self._request(query)
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/__init__.py", line 118, in _request
response = yql.YQLQuery().execute(query)
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/yql.py", line 61, in execute
'env': DATATABLES_URL
File "/usr/lib/python3.6/urllib/request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "/usr/lib/python3.6/urllib/request.py", line 526, in open
response = self._open(req, data)
File "/usr/lib/python3.6/urllib/request.py", line 544, in _open
'_open', req)
File "/usr/lib/python3.6/urllib/request.py", line 504, in _call_chain
result = func(*args)
File "/usr/lib/python3.6/urllib/request.py", line 1361, in https_open
context=self._context, check_hostname=self._check_hostname)
File "/usr/lib/python3.6/urllib/request.py", line 1320, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno -2] Name or service not known>
i have a no idea about this error, please help me to solve this problem. Thank you....
The package is broken. See for example, this issue on GitHub: https://github.com/lukaszbanasiak/yahoo-finance/issues/148
Your best bet would be to indicate on GitHub that the package is not working and perhaps the author is able to help you further.
I'm trying to install an application developed by someone else and when running a manage.py command I'm getting this error, which I do not understand. I wonder if anyone can tell me where to look to investigate first - thank you.
POST http://elasticsearch:9200/_bulk?refresh=true [status:N/A request:0.001s]
Traceback (most recent call last):
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/django/db/models/query.py", line 486, in get_or_create
return self.get(**lookup), False
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/djmoney/models/managers.py", line 209, in wrapper
queryset = func(*args, **kwargs)
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/django/db/models/query.py", line 399, in get
self.model._meta.object_name
providers.models.Organisation.DoesNotExist: Organisation matching query does not exist.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connection.py", line 160, in _new_conn
(self._dns_host, self.port), self.timeout, **extra_kw)
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/util/connection.py", line 57, in create_connection
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
File "/usr/lib/python3.6/socket.py", line 745, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/elasticsearch/connection/http_urllib3.py", line 95, in perform_request
response = self.pool.urlopen(method, url, body, retries=False, headers=self.headers, **kw)
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connectionpool.py", line 641, in urlopen
_stacktrace=sys.exc_info()[2])
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/util/retry.py", line 344, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/packages/six.py", line 686, in reraise
raise value
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connectionpool.py", line 603, in urlopen
chunked=chunked)
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connectionpool.py", line 355, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib/python3.6/http/client.py", line 1254, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1300, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1249, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1036, in _send_output
self.send(msg)
File "/usr/lib/python3.6/http/client.py", line 974, in send
self.connect()
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connection.py", line 183, in connect
conn = self._new_conn()
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connection.py", line 169, in _new_conn
self, "Failed to establish a new connection: %s" % e)
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7f506c988eb8>: Failed to establish a new connection: [Errno -2] Name or service not known
EDIT
Added manage.py as requested
#!/usr/bin/env python
import os
import sys
if __name__ == "__main__":
os.environ.setdefault('DEPLOY_ENV', 'local')
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings.%s' % os.environ['DEPLOY_ENV'])
from django.core.management import execute_from_command_line # noqa
execute_from_command_line(sys.argv)
As Goran said, I was trying to connect to elasticsearch:9200 but testing localhost:9200
Here's my code:
import yweather
client = yweather.Client()
id = client.fetch_woeid("Beijing, China")
beijing_weather = client.fetch_weather(id)
Nice and simple... right out of yweather's documentation, yet I get an error:
> Traceback (most recent call last):
File "C:\Users\DSHenderson\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 1318, in do_open
encode_chunked=req.has_header('Transfer-encoding'))
File "C:\Users\DSHenderson\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\DSHenderson\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\DSHenderson\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\DSHenderson\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 1026, in _send_output
self.send(msg)
File "C:\Users\DSHenderson\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 964, in send
self.connect()
File "C:\Users\DSHenderson\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 936, in connect
(self.host,self.port), self.timeout, self.source_address)
File "C:\Users\DSHenderson\AppData\Local\Programs\Python\Python36\lib\socket.py", line 704, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
File "C:\Users\DSHenderson\AppData\Local\Programs\Python\Python36\lib\socket.py", line 743, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno 11001] getaddrinfo failed
>
> During handling of the above exception, another exception occurred:
>
> Traceback (most recent call last): File
> "C:/Users/DSHenderson/PycharmProjects/WeatherAPI/Main.py", line 4, in
> <module>
> beijing_weather = client.fetch_weather(id) File "C:\Users\DSHenderson\AppData\Local\Programs\Python\Python36\lib\site-packages\yweather.py",
> line 180, in fetch_weather
> rss = self._fetch_xml(url) File "C:\Users\DSHenderson\AppData\Local\Programs\Python\Python36\lib\site-packages\yweather.py",
> line 344, in _fetch_xml
> with contextlib.closing(urlopen(url)) as f: File "C:\Users\DSHenderson\AppData\Local\Programs\Python\Python36\lib\urllib\request.py",
> line 223, in urlopen
> return opener.open(url, data, timeout) File "C:\Users\DSHenderson\AppData\Local\Programs\Python\Python36\lib\urllib\request.py",
> line 526, in open
> response = self._open(req, data) File "C:\Users\DSHenderson\AppData\Local\Programs\Python\Python36\lib\urllib\request.py",
> line 544, in _open
> '_open', req) File "C:\Users\DSHenderson\AppData\Local\Programs\Python\Python36\lib\urllib\request.py",
> line 504, in _call_chain
> result = func(*args) File "C:\Users\DSHenderson\AppData\Local\Programs\Python\Python36\lib\urllib\request.py",
> line 1346, in http_open
> return self.do_open(http.client.HTTPConnection, req) File "C:\Users\DSHenderson\AppData\Local\Programs\Python\Python36\lib\urllib\request.py",
> line 1320, in do_open
> raise URLError(err) urllib.error.URLError: <urlopen error [Errno 11001] getaddrinfo failed>
That's kind of frustrating. Any ideas?
Edit: converted the traceback to "code" for readability.
Edit2: using forecastiopy now, seems to get the job done in the same fashion.
I'm trying to visit a url, each time using a different proxy, from a file, until all proxies have been used. Getting errors:
socket.gaierror: [Errno 11004] getaddrinfo failed
urllib.error.URLError: <urlopen error [Errno 11004] getaddrinfo failed>
proxies1 file has 300 lines like this:
34.195.146.122:80
import urllib.request
my_file = open("proxies1","r")
for n in my_file:
proxy = urllib.request.ProxyHandler({'http': 'n'})
opener = urllib.request.build_opener(proxy)
urllib.request.install_opener(opener)
urllib.request.urlretrieve('http://www.google.com')
Getting the following output:
Traceback (most recent call last):
File "C:\Users\user\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 1318, in do_open
encode_chunked=req.has_header('Transfer-encoding'))
File "C:\Users\user\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\user\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\user\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\user\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1026, in _send_output
self.send(msg)
File "C:\Users\user\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 964, in send
self.connect()
File "C:\Users\user\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 936, in connect
(self.host,self.port), self.timeout, self.source_address)
File "C:\Users\user\AppData\Local\Programs\Python\Python36-32\lib\socket.py", line 704, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
File "C:\Users\user\AppData\Local\Programs\Python\Python36-32\lib\socket.py", line 743, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno 11004] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "p3.py", line 12, in <module>
urllib.request.urlretrieve('http://www.google.com')
File "C:\Users\user\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 248, in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
File "C:\Users\user\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "C:\Users\user\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 526, in open
response = self._open(req, data)
File "C:\Users\user\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 544, in _open
'_open', req)
File "C:\Users\user\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 504, in _call_chain
result = func(*args)
File "C:\Users\user\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 1346, in http_open
return self.do_open(http.client.HTTPConnection, req)
File "C:\Users\user\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 1320, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 11004] getaddrinfo failed>
I am using Python 3.4 on Windows 7, and I am trying to test if a proxy allows or denies connections to particular websites by using a python script.
I am using the code below:
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError, urllib
conf = "http://{}:{}#{}".format(login, password, proxy)
supp = urllib.request.ProxyHandler({"http": conf})
auth = urllib.request.HTTPBasicAuthHandler()
open = urllib.request.build_opener(supp, auth, urllib.request.HTTPHandler)
urllib.request.install_opener(open)
response = urlopen(Request("http://www.google.com"))
I get no errors when executing the code above, however as soon as I switch the URL to an HTTPS one (for example, https://www.google.com), I get the following error:
C:\Python34\python.exe test_url.py
Traceback (most recent call last):
File "C:\Python34\lib\urllib\request.py", line 1182, in do_open
h.request(req.get_method(), req.selector, req.data, headers)
File "C:\Python34\lib\http\client.py", line 1088, in request
self._send_request(method, url, body, headers)
File "C:\Python34\lib\http\client.py", line 1126, in _send_request
self.endheaders(body)
File "C:\Python34\lib\http\client.py", line 1084, in endheaders
self._send_output(message_body)
File "C:\Python34\lib\http\client.py", line 922, in _send_output
self.send(msg)
File "C:\Python34\lib\http\client.py", line 857, in send
self.connect()
File "C:\Python34\lib\http\client.py", line 1223, in connect
super().connect()
File "C:\Python34\lib\http\client.py", line 834, in connect
self.timeout, self.source_address)
File "C:\Python34\lib\socket.py", line 494, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
File "C:\Python34\lib\socket.py", line 533, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno 11004] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "test.py", line 14, in <module>
response = urlopen(Request("https://www.google.com"))
File "C:\Python34\lib\urllib\request.py", line 161, in urlopen
return opener.open(url, data, timeout)
File "C:\Python34\lib\urllib\request.py", line 463, in open
response = self._open(req, data)
File "C:\Python34\lib\urllib\request.py", line 481, in _open
'_open', req)
File "C:\Python34\lib\urllib\request.py", line 441, in _call_chain
result = func(*args)
File "C:\Python34\lib\urllib\request.py", line 1225, in https_open
context=self._context, check_hostname=self._check_hostname)
File "C:\Python34\lib\urllib\request.py", line 1184, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 11004] getaddrinfo failed>
Any ideas why my code is only working with HTTP websites?
You need to specify the proxy handler for HTTPS separately, as it is a different protocol to HTTP. So the ProxyHandler line should be changed to:
supp = urllib.request.ProxyHandler({"http": conf, "https": conf})