This code fails on my Ubuntu, but works well on other hosts.
>>> from urllib2 import urlopen
>>> urlopen("https://courtapps.utcourts.gov/XchangeWEB/login")
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/lib/python2.7/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.7/urllib2.py", line 400, in open
response = self._open(req, data)
File "/usr/lib/python2.7/urllib2.py", line 418, in _open
'_open', req)
File "/usr/lib/python2.7/urllib2.py", line 378, in _call_chain
result = func(*args)
File "/usr/lib/python2.7/urllib2.py", line 1215, in https_open
return self.do_open(httplib.HTTPSConnection, req)
File "/usr/lib/python2.7/urllib2.py", line 1177, in do_open
raise URLError(err)
urllib2.URLError: <urlopen error [Errno 1] _ssl.c:504: error:1408F119:SSL
routines:SSL3_GET_RECORD:decryption failed or bad record mac>
What is the reason and how to fix this? I feel something is wrong with OpenSSL, have v1.0.1 installed.
Custom SSLv3 handler solves the problem:
import httplib, ssl, urllib2, socket
class HTTPSConnectionV3(httplib.HTTPSConnection):
def __init__(self, *args, **kwargs):
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
def connect(self):
sock = socket.create_connection((self.host, self.port), self.timeout)
if self._tunnel_host:
self.sock = sock
self._tunnel()
try:
self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv3)
except ssl.SSLError, e:
print("Trying SSLv3.")
self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23)
class HTTPSHandlerV3(urllib2.HTTPSHandler):
def https_open(self, req):
return self.do_open(HTTPSConnectionV3, req)
# install opener
urllib2.install_opener(urllib2.build_opener(HTTPSHandlerV3()))
if __name__ == "__main__":
r = urllib2.urlopen("https://courtapps.utcourts.gov/XchangeWEB/login")
print(r.read())
Related
I am trying to bypass the SSL Certificate and use the URL in Python,
All HTTPS sites are getting the same error
Kindly suggest how can it get resolved. Thanks in Advance.
Code:
import urllib.request as ur
import urllib.parse, urllib.error, ssl
url_is = 'https://finance.yahoo.com'
url_google = 'https://www.google.co.in'
req = ur.Request(url_google)
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
info = ur.urlopen(req, context=ctx).read()
Message.Chat.SendMessage ("" + info)
Error:
Traceback (most recent call last):
File "/usr/lib/python3.8/urllib/request.py", line 1354, in do_open
h.request(req.get_method(), req.selector, req.data, headers,
File "/usr/lib/python3.8/http/client.py", line 1256, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.8/http/client.py", line 1302, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.8/http/client.py", line 1251, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.8/http/client.py", line 1011, in _send_output
self.send(msg)
File "/usr/lib/python3.8/http/client.py", line 951, in send
self.connect()
File "/usr/lib/python3.8/http/client.py", line 1418, in connect
super().connect()
File "/usr/lib/python3.8/http/client.py", line 922, in connect
self.sock = self._create_connection(
File "/usr/lib/python3.8/socket.py", line 787, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
File "/usr/lib/python3.8/socket.py", line 918, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -3] Temporary failure in name resolution
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "main.py", line 10, in <module>
info = ur.urlopen(req, context=gcontext).read()
File "/usr/lib/python3.8/urllib/request.py", line 222, in urlopen
return opener.open(url, data, timeout)
File "/usr/lib/python3.8/urllib/request.py", line 525, in open
response = self._open(req, data)
File "/usr/lib/python3.8/urllib/request.py", line 542, in _open
result = self._call_chain(self.handle_open, protocol, protocol +
File "/usr/lib/python3.8/urllib/request.py", line 502, in _call_chain
result = func(*args)
File "/usr/lib/python3.8/urllib/request.py", line 1397, in https_open
return self.do_open(http.client.HTTPSConnection, req,
File "/usr/lib/python3.8/urllib/request.py", line 1357, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno -3] Temporary failure in name resolution>
Example is available at https://onlinegdb.com/rdj-HSEFAz
I have also tried with the below code but the same issue.
from bs4 import BeautifulSoup
import urllib.request as ur
import urllib.parse, urllib.error, ssl
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
url_is = 'https://finance.yahoo.com'
url_google = 'https://www.google.co.in'
read_data = urllib.request.urlopen(url_google, context=ctx).read()
soup_is= BeautifulSoup(read_data,'lxml')
Temporary failure in name resolution
This means that no IP address for the given hostname can be found. This is completely unrelated to SSL and thus no "bypass SSL certificate" will help.
The problem is instead that DNS does not properly work in the software environment where this code is run. This needs to be fixed in this environment (no details given, so no help can be provided) and not in the code.
I have tested that with a working DNS the code works fine.
I'm trying to run a simple code as below from my workplace,
proxy = urllib2.ProxyHandler({'http': 'proxy.examply.com:8080'})
opener = build_opener(proxy)
resp = opener.open('http://www.solver.io')
But it throws below error-
Traceback (most recent call last):
File "<pyshell#11>", line 1, in <module>
response = urllib2.urlopen('http://www.solver.io')
File "C:\Python27\lib\urllib2.py", line 154, in urlopen
return opener.open(url, data, timeout)
File "C:\Python27\lib\urllib2.py", line 437, in open
response = meth(req, response)
File "C:\Python27\lib\urllib2.py", line 550, in http_response
'http', request, response, code, msg, hdrs)
File "C:\Python27\lib\urllib2.py", line 469, in error
result = self._call_chain(*args)
File "C:\Python27\lib\urllib2.py", line 409, in _call_chain
result = func(*args)
File "C:\Python27\lib\urllib2.py", line 656, in http_error_302
return self.parent.open(new, timeout=req.timeout)
File "C:\Python27\lib\urllib2.py", line 431, in open
response = self._open(req, data)
File "C:\Python27\lib\urllib2.py", line 449, in _open
'_open', req)
File "C:\Python27\lib\urllib2.py", line 409, in _call_chain
result = func(*args)
File "C:\Python27\lib\urllib2.py", line 1240, in https_open
context=self._context)
File "C:\Python27\lib\urllib2.py", line 1197, in do_open
raise URLError(err)
URLError: <urlopen error [Errno 10060] A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond>
I've tried pip install , aws, boto which work just fine behind the proxy.
I've even tried this solutions on stackoverflow but didnt work
Proxy with urllib2
how to resolve this please?
I am trying to process urllib request over tor. it's worked for me on both computers well now I don't know why but it don't work any more on one of the computers. I know that there are alot of posts about urllib over tor but it dosen't work for me.
The code with example site for checks:
import socket
import socks
def create_connection(address, timeout=None, source_address=None):
sock = socks.socksocket()
sock.connect(address)
return sock
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, '127.0.0.1', 9050, True)
socket.socket = socks.socksocket
socket.create_connection = create_connection
import urllib2
print urllib2.urlopen("http://bm26rwk32m7u7rec.onion/index.php").read()
exception traceback:
Traceback (most recent call last):
File "check.py", line 15, in <module>
print urllib2.urlopen("http://bm26rwk32m7u7rec.onion/index.php").read()
File "/usr/local/lib/python2.7/urllib2.py", line 127, in urlopen
return _opener.open(url, data, timeout)
File "/usr/local/lib/python2.7/urllib2.py", line 404, in open
response = self._open(req, data)
File "/usr/local/lib/python2.7/urllib2.py", line 422, in _open
'_open', req)
File "/usr/local/lib/python2.7/urllib2.py", line 382, in _call_chain
result = func(*args)
File "/usr/local/lib/python2.7/urllib2.py", line 1214, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/usr/local/lib/python2.7/urllib2.py", line 1181, in do_open
h.request(req.get_method(), req.get_selector(), req.data, headers)
File "/usr/local/lib/python2.7/httplib.py", line 973, in request
self._send_request(method, url, body, headers)
File "/usr/local/lib/python2.7/httplib.py", line 1007, in _send_request
self.endheaders(body)
File "/usr/local/lib/python2.7/httplib.py", line 969, in endheaders
self._send_output(message_body)
File "/usr/local/lib/python2.7/httplib.py", line 829, in _send_output
self.send(msg)
File "/usr/local/lib/python2.7/httplib.py", line 791, in send
self.connect()
File "/usr/local/lib/python2.7/httplib.py", line 772, in connect
self.timeout, self.source_address)
File "check.py", line 5, in create_connection
sock.connect(address)
File "/home/lior/code/socks.py", line 369, in connect
self.__negotiatesocks5(destpair[0],destpair[1])
File "/home/lior/code/socks.py", line 236, in __negotiatesocks5
raise Socks5Error(ord(resp[1]),_generalerrors[ord(resp[1])])
TypeError: __init__() takes exactly 2 arguments (3 given)
tried also this code:
import urllib2, socks, socket
from stem import Signal
from stem.control import Controller
old_socket = socket.socket
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, "127.0.0.1", 9050)
socket.socket = socks.socksocket
def newI():
socket.socket = old_socket # don't use proxy
with Controller.from_port(port=9051) as controller:
controller.authenticate()
controller.signal(Signal.NEWNYM)
# set up the proxy again
socket.socket = socks.socksocket
newI()
headers = {'User-Agent': 'Mozilla/3.0 (x86 [en] Windows NT 5.1; Sun)'}
req = urllib2.Request('https://google.com', None, headers)
response = urllib2.urlopen(req)
html = response.read()
newI()
got:
stem : INFO Error while receiving a control message (SocketClosed): empty socket content
I'm trying to use python to login and download some files, using the code:
import sys
import urllib
import urllib2
import httplib, ssl, socket
class HTTPSConnectionV3(httplib.HTTPSConnection):
def __init__(self, *args, **kwargs):
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
def connect(self):
sock = socket.create_connection((self.host, self.port), self.timeout)
if self._tunnel_host:
self.sock = sock
self._tunnel()
try:
self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv3)
except ssl.SSLError, e:
print("Trying SSLv3.")
#self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_TLSv1)
self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23)
class HTTPSHandlerV3(urllib2.HTTPSHandler):
def https_open(self, req):
return self.do_open(HTTPSConnectionV3, req)
# install opener
urllib2.install_opener(urllib2.build_opener(HTTPSHandlerV3()))
if __name__ == "__main__":
##fill the login form
query={}
query['username']='USER'
query['password']='007'
query['submit']='Submit'
if len(sys.argv) != 2:
print >> sys.stderr, "missing date"
sys.exit()
#submit the form
#http_req = urllib2.Request(url='https://www.connect2nse.com/iislNet', data=urllib.urlencode(query))
http_req = urllib2.Request(url='https://www.connect2nse.com/iislNet/Login.jsp', data=urllib.urlencode(query))
#http_req = urllib2.Request(url='https://www.connect2nse.com/iislNet/index.html', data=urllib.urlencode(query))
webpage = urllib2.urlopen(http_req)
webpage_headers = webpage.info()
#extract the cookie
cookie = webpage_headers['Set-Cookie'].split(';', 1)[0]
print >> sys.stderr, "Set-Cookie:", cookie
http_req = urllib2.Request(url='https://connect2nse.com/iislNet/MY.jsp', headers={'Cookie': cookie})
webpage = urllib2.urlopen(http_req)
I get the following error
Trying SSLv3.
Traceback (most recent call last):
File "MYSCRIPT.py", line 51, in <module>
webpage = urllib2.urlopen(http_req)
File "/usr/lib/python2.7/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.7/urllib2.py", line 400, in open
response = self._open(req, data)
File "/usr/lib/python2.7/urllib2.py", line 418, in _open
'_open', req)
File "/usr/lib/python2.7/urllib2.py", line 378, in _call_chain
result = func(*args)
File "MYSCRIPT.py", line 27, in https_open
return self.do_open(HTTPSConnectionV3, req)
File "/usr/lib/python2.7/urllib2.py", line 1177, in do_open
raise URLError(err)
urllib2.URLError: <urlopen error [Errno 8] _ssl.c:504: EOF occurred in violation of protocol>
Although this was working fine till last month but now it gives the above error
I have already tried This link but to no avail.
Using the requests in python I use
r = requests.get('https://www.connect2nse.com/iislNet/Login.jsp', auth=('USER', '007'))
r.status_code
401
But I'm not sure how to proceed using requests also.
Trying #Antti Haapala's solution doesnt work for me, tried it on 2.7.3&6
try:
# self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23)
self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_TLSv1)
except ssl.SSLError, e:
print("Trying SSLv3.",e)
# self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=5)
self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23)
TLSv1 and SSLv23 both give similar errors.
error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure' and
error:14094410:SSL routines:SSL3_READ_BYTES:sslv3 alert handshake failure'
#J.F. Sebastian , thanks but I'm 100% sure that login and password are correct, because I use the same while using in chrome. When I use TLSv1_2, I get the below error
python MYSCRIPT.py 090315
Trying SSLv3.
Traceback (most recent call last):
File "MYSCRIPT.py", line 51, in <module>
webpage = urllib2.urlopen(http_req)
File "/usr/lib/python2.7/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.7/urllib2.py", line 400, in open
response = self._open(req, data)
File "/usr/lib/python2.7/urllib2.py", line 418, in _open
'_open', req)
File "/usr/lib/python2.7/urllib2.py", line 378, in _call_chain
result = func(*args)
File "MYSCRIPT.py", line 27, in https_open
return self.do_open(HTTPSConnectionV3, req)
File "/usr/lib/python2.7/urllib2.py", line 1174, in do_open
h.request(req.get_method(), req.get_selector(), req.data, headers)
File "/usr/lib/python2.7/httplib.py", line 958, in request
self._send_request(method, url, body, headers)
File "/usr/lib/python2.7/httplib.py", line 992, in _send_request
self.endheaders(body)
File "/usr/lib/python2.7/httplib.py", line 954, in endheaders
self._send_output(message_body)
File "/usr/lib/python2.7/httplib.py", line 814, in _send_output
self.send(msg)
File "/usr/lib/python2.7/httplib.py", line 776, in send
self.connect()
File "MYSCRIPT.py", line 22, in connect
self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_TLSv1_2)
AttributeError: 'module' object has no attribute 'PROTOCOL_TLSv1_2'
The site published a notification of compatible browser settings, after which the script started malfunctioning this link shows compatibility, you need to click on NEW SSL DOC
Your code forces the connection to use SSL 3 protocol always, SSL 3 has been superseded in the last millennium by TLSv1.
In Pythons < 2.7.9 (2.7.8), you should choose ssl.PROTOCOL_SSLv23, which will support the highest possible TLS number supported by the OpenSSL library. It specifically in current OpenSSL versions means that SSLv3, TLSv1, TLSv1.1, and TLSv1.2 are supported. Unlike the flag says, SSLv2 will not be accepted with recent versions of OpenSSL.
Thus we get:
def connect(self):
sock = socket.create_connection((self.host, self.port), self.timeout)
if self._tunnel_host:
self.sock = sock
self._tunnel()
self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
ssl_version=ssl.PROTOCOL_SSLv23)
If requests connects, then you might want to use that instead. Your authorization should be sent as form values, and as a POST request:
data = {}
data['username']='USER'
data['password']='007'
data['submit']='Submit'
r = requests.post("https://www.connect2nse.com/iislNet/Login.jsp", data=data)
ssl.PROTOCOL_TLSv1_2 appeared in Python 2.7.9 and 3.4, but is also available some backports even before that version number.
However, if PROTOCOL_TLSv1_2 is not in ssl module, it also means that TLS 1.2 only cannot be used in Python even with the hardcoded protocol constant (5) - The reason is quite obvious from the _ssl source code - the integer is only meaningful to the Python extension module, and it is used to choose the actual constructor method.
In Python 2.7.9 there will be a SSLContext object that can be used to set flags on for SSL socket creation; in there one can at least try to monkey_patch to be future proof, but it is not possible with versions that do not also have the TLSv1.2 patch.
An example code on 2.7.9 on how to disable SSL 2 and 3, TLSv1.0, TLSv1.1, TLSv1.2, forcing to use the hypothetical TLSv1.3(?):
ssl_sock = ssl.wrap_socket(sock, ssl_version=ssl.PROTOCOL_SSLv23)
ssl_sock.context.options |= ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 \
| ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 | ssl.OP_NO_TLSv1_2
My code
conn = __get_s3_connection(s3_values.get('accessKeyId'), s3_values.get('secretAccessKey'))
key = s3_values.get('proposal_key') + proposal_unique_id + s3_values.get('proposal_append_path')
request = urllib2.Request(conn.generate_url(s3_values.get('expires_in'), 'GET', bucket=s3_values.get('bucket'), key=key))
request.add_header('Accept-encoding', 'gzip')
response = urllib2.urlopen(request)
The url looks like https://production.myorg.s3.amazonaws.com/key/document.xml.gz?Signature=signature%3D&Expires=1349462207&AWSAccessKeyId=accessId
This method was working fine until 1 hour back, but when I run the same program, it throws
Traceback (most recent call last):
File "/Users/hhimanshu/IdeaProjects/analytics/src/utilities/documentReader.py", line 145, in <module>
main()
File "/Users/hhimanshu/IdeaProjects/analytics/src/utilities/documentReader.py", line 141, in main
x = get_proposal_data_from_s3('documentId')
File "/Users/hhimanshu/IdeaProjects/analytics/src/utilities/documentReader.py", line 54, in get_proposal_data_from_s3
response = urllib2.urlopen(request)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 392, in open
response = self._open(req, data)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 410, in _open
'_open', req)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 370, in _call_chain
result = func(*args)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 1194, in https_open
return self.do_open(httplib.HTTPSConnection, req)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 1161, in do_open
raise URLError(err)
urllib2.URLError: <urlopen error [Errno 6] _ssl.c:503: TLS/SSL connection has been closed>
What could be the reason? How can I avoid this situation?
This was because of intermittent internet connection. Resolved on it own