I am trying to check the response of the URL same as the domain record from the WHOIS database.
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
The code:
def abnormal_url(url):
response = requests.get(url,verify=False)
domainname = urlparse(url).netloc
domain = whois.whois(domainname)
try:
if response.text == domain:
return 0 # legitimate
else:
return 1 # phishing
except:
return 1 # phishing
Append to dataframe:
df['abnormal url'] = df['url'].apply(lambda i: abnormal_url(i))
Error found:
gaierror Traceback (most recent call last)
File D:\anaconda3\lib\site-packages\urllib3\connection.py:174, in HTTPConnection._new_conn(self)
173 try:
--> 174 conn = connection.create_connection(
175 (self._dns_host, self.port), self.timeout, **extra_kw
176 )
178 except SocketTimeout:
File D:\anaconda3\lib\site-packages\urllib3\util\connection.py:72, in create_connection(address, timeout, source_address, socket_options)
68 return six.raise_from(
69 LocationParseError(u"'%s', label empty or too long" % host), None
70 )
---> 72 for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
73 af, socktype, proto, canonname, sa = res
File D:\anaconda3\lib\socket.py:954, in getaddrinfo(host, port, family, type, proto, flags)
953 addrlist = []
--> 954 for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
955 af, socktype, proto, canonname, sa = res
gaierror: [Errno 11001] getaddrinfo failed
During handling of the above exception, another exception occurred:
NewConnectionError Traceback (most recent call last)
File D:\anaconda3\lib\site-packages\urllib3\connectionpool.py:703, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
702 # Make the request on the httplib connection object.
--> 703 httplib_response = self._make_request(
704 conn,
705 method,
706 url,
707 timeout=timeout_obj,
708 body=body,
709 headers=headers,
710 chunked=chunked,
711 )
713 # If we're going to release the connection in ``finally:``, then
714 # the response doesn't need to know about the connection. Otherwise
715 # it will also try to release it and we'll have a double-release
716 # mess.
File D:\anaconda3\lib\site-packages\urllib3\connectionpool.py:386, in HTTPConnectionPool._make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
385 try:
--> 386 self._validate_conn(conn)
387 except (SocketTimeout, BaseSSLError) as e:
388 # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.
File D:\anaconda3\lib\site-packages\urllib3\connectionpool.py:1040, in HTTPSConnectionPool._validate_conn(self, conn)
1039 if not getattr(conn, "sock", None): # AppEngine might not have `.sock`
-> 1040 conn.connect()
1042 if not conn.is_verified:
File D:\anaconda3\lib\site-packages\urllib3\connection.py:358, in HTTPSConnection.connect(self)
356 def connect(self):
357 # Add certificate verification
--> 358 self.sock = conn = self._new_conn()
359 hostname = self.host
File D:\anaconda3\lib\site-packages\urllib3\connection.py:186, in HTTPConnection._new_conn(self)
185 except SocketError as e:
--> 186 raise NewConnectionError(
187 self, "Failed to establish a new connection: %s" % e
188 )
190 return conn
NewConnectionError: <urllib3.connection.HTTPSConnection object at 0x0000023F7EA21520>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed
During handling of the above exception, another exception occurred:
MaxRetryError Traceback (most recent call last)
File D:\anaconda3\lib\site-packages\requests\adapters.py:440, in HTTPAdapter.send(self, request, stream, timeout, verify, cert, proxies)
439 if not chunked:
--> 440 resp = conn.urlopen(
441 method=request.method,
442 url=url,
443 body=request.body,
444 headers=request.headers,
445 redirect=False,
446 assert_same_host=False,
447 preload_content=False,
448 decode_content=False,
449 retries=self.max_retries,
450 timeout=timeout
451 )
453 # Send the request.
454 else:
File D:\anaconda3\lib\site-packages\urllib3\connectionpool.py:785, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
783 e = ProtocolError("Connection aborted.", e)
--> 785 retries = retries.increment(
786 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
787 )
788 retries.sleep()
File D:\anaconda3\lib\site-packages\urllib3\util\retry.py:592, in Retry.increment(self, method, url, response, error, _pool, _stacktrace)
591 if new_retry.is_exhausted():
--> 592 raise MaxRetryError(_pool, url, error or ResponseError(cause))
594 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
MaxRetryError: HTTPSConnectionPool(host='www.list.tmall.com', port=443): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x0000023F7EA21520>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed'))
During handling of the above exception, another exception occurred:
ConnectionError Traceback (most recent call last)
Input In [16], in <cell line: 1>()
----> 1 df['abnormal url'] = df['url'].apply(lambda i: abnormal_url(i))
File D:\anaconda3\lib\site-packages\pandas\core\series.py:4433, in Series.apply(self, func, convert_dtype, args, **kwargs)
4323 def apply(
4324 self,
4325 func: AggFuncType,
(...)
4328 **kwargs,
4329 ) -> DataFrame | Series:
4330 """
4331 Invoke function on values of Series.
4332
(...)
4431 dtype: float64
4432 """
-> 4433 return SeriesApply(self, func, convert_dtype, args, kwargs).apply()
File D:\anaconda3\lib\site-packages\pandas\core\apply.py:1082, in SeriesApply.apply(self)
1078 if isinstance(self.f, str):
1079 # if we are a string, try to dispatch
1080 return self.apply_str()
-> 1082 return self.apply_standard()
File D:\anaconda3\lib\site-packages\pandas\core\apply.py:1137, in SeriesApply.apply_standard(self)
1131 values = obj.astype(object)._values
1132 # error: Argument 2 to "map_infer" has incompatible type
1133 # "Union[Callable[..., Any], str, List[Union[Callable[..., Any], str]],
1134 # Dict[Hashable, Union[Union[Callable[..., Any], str],
1135 # List[Union[Callable[..., Any], str]]]]]"; expected
1136 # "Callable[[Any], Any]"
-> 1137 mapped = lib.map_infer(
1138 values,
1139 f, # type: ignore[arg-type]
1140 convert=self.convert_dtype,
1141 )
1143 if len(mapped) and isinstance(mapped[0], ABCSeries):
1144 # GH#43986 Need to do list(mapped) in order to get treated as nested
1145 # See also GH#25959 regarding EA support
1146 return obj._constructor_expanddim(list(mapped), index=obj.index)
File D:\anaconda3\lib\site-packages\pandas\_libs\lib.pyx:2870, in pandas._libs.lib.map_infer()
Input In [16], in <lambda>(i)
----> 1 df['abnormal url'] = df['url'].apply(lambda i: abnormal_url(i))
Input In [15], in abnormal_url(url)
1 def abnormal_url(url):
----> 2 response = requests.get(url,verify=False)
3 domainname = urlparse(url).netloc
4 domain = whois.whois(domainname)
File D:\anaconda3\lib\site-packages\requests\api.py:75, in get(url, params, **kwargs)
64 def get(url, params=None, **kwargs):
65 r"""Sends a GET request.
66
67 :param url: URL for the new :class:`Request` object.
(...)
72 :rtype: requests.Response
73 """
---> 75 return request('get', url, params=params, **kwargs)
File D:\anaconda3\lib\site-packages\requests\api.py:61, in request(method, url, **kwargs)
57 # By using the 'with' statement we are sure the session is closed, thus we
58 # avoid leaving sockets open which can trigger a ResourceWarning in some
59 # cases, and look like a memory leak in others.
60 with sessions.Session() as session:
---> 61 return session.request(method=method, url=url, **kwargs)
File D:\anaconda3\lib\site-packages\requests\sessions.py:529, in Session.request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
524 send_kwargs = {
525 'timeout': timeout,
526 'allow_redirects': allow_redirects,
527 }
528 send_kwargs.update(settings)
--> 529 resp = self.send(prep, **send_kwargs)
531 return resp
File D:\anaconda3\lib\site-packages\requests\sessions.py:645, in Session.send(self, request, **kwargs)
642 start = preferred_clock()
644 # Send the request
--> 645 r = adapter.send(request, **kwargs)
647 # Total elapsed time of the request (approximately)
648 elapsed = preferred_clock() - start
File D:\anaconda3\lib\site-packages\requests\adapters.py:519, in HTTPAdapter.send(self, request, stream, timeout, verify, cert, proxies)
515 if isinstance(e.reason, _SSLError):
516 # This branch is for urllib3 v1.22 and later.
517 raise SSLError(e, request=request)
--> 519 raise ConnectionError(e, request=request)
521 except ClosedPoolError as e:
522 raise ConnectionError(e, request=request)
ConnectionError: HTTPSConnectionPool(host='www.list.tmall.com', port=443): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x0000023F7EA21520>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed'))
The connection could not be established because the site can't be reached.
Just execute the get request inside your try/except and it will work.
def abnormal_url(url):
domainname = urlparse(url).netloc
domain = whois.whois(domainname)
try:
response = requests.get(url,verify=False)
if response.text == domain:
return 0 # legitimate
else:
return 1 # phishing
except:
return 1 # phishing
Related
I'm trying to use an API to download our account data from a third party analytics company. There's a lot to download, and they only offer a throttled API to do it so we're working with batches of 100 records per request with at most four requests per minute. I keep getting three errors along the way, at irregular intervals (ProtocolError, ConnectionError, ConnectionResetError). I believe this is a serverside issue, so I'm just trying to work with catching the error. Catching it isn't working though, as it ignores the try/except block.
try:
response = requests.get(url+collection, headers=headers, params=params).json()
except (ConnectionResetError, ConnectionError) as err:
request_errors.append(collection)
response = requests.get(url+collection, headers=headers, params=params).json()
It's worth noting that you don't see ProtocolError in there. When I try to catch ProtocolError, I get a response saying that name isn't recognized.
---------------------------------------------------------------------------
ConnectionResetError Traceback (most recent call last)
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py:703, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
702 # Make the request on the httplib connection object.
--> 703 httplib_response = self._make_request(
704 conn,
705 method,
706 url,
707 timeout=timeout_obj,
708 body=body,
709 headers=headers,
710 chunked=chunked,
711 )
713 # If we're going to release the connection in ``finally:``, then
714 # the response doesn't need to know about the connection. Otherwise
715 # it will also try to release it and we'll have a double-release
716 # mess.
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py:386, in HTTPConnectionPool._make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
385 try:
--> 386 self._validate_conn(conn)
387 except (SocketTimeout, BaseSSLError) as e:
388 # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py:1042, in HTTPSConnectionPool._validate_conn(self, conn)
1041 if not getattr(conn, "sock", None): # AppEngine might not have `.sock`
-> 1042 conn.connect()
1044 if not conn.is_verified:
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connection.py:414, in HTTPSConnection.connect(self)
412 context.load_default_certs()
--> 414 self.sock = ssl_wrap_socket(
415 sock=conn,
416 keyfile=self.key_file,
417 certfile=self.cert_file,
418 key_password=self.key_password,
419 ca_certs=self.ca_certs,
420 ca_cert_dir=self.ca_cert_dir,
421 ca_cert_data=self.ca_cert_data,
422 server_hostname=server_hostname,
423 ssl_context=context,
424 tls_in_tls=tls_in_tls,
425 )
427 # If we're using all defaults and the connection
428 # is TLSv1 or TLSv1.1 we throw a DeprecationWarning
429 # for the host.
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\util\ssl_.py:449, in ssl_wrap_socket(sock, keyfile, certfile, cert_reqs, ca_certs, server_hostname, ssl_version, ciphers, ssl_context, ca_cert_dir, key_password, ca_cert_data, tls_in_tls)
448 if send_sni:
--> 449 ssl_sock = _ssl_wrap_socket_impl(
450 sock, context, tls_in_tls, server_hostname=server_hostname
451 )
452 else:
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\util\ssl_.py:493, in _ssl_wrap_socket_impl(sock, ssl_context, tls_in_tls, server_hostname)
492 if server_hostname:
--> 493 return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
494 else:
File ~\AppData\Local\Programs\Python\Python39\lib\ssl.py:501, in SSLContext.wrap_socket(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)
495 def wrap_socket(self, sock, server_side=False,
496 do_handshake_on_connect=True,
497 suppress_ragged_eofs=True,
498 server_hostname=None, session=None):
499 # SSLSocket class handles server_hostname encoding before it calls
500 # ctx._wrap_socket()
--> 501 return self.sslsocket_class._create(
502 sock=sock,
503 server_side=server_side,
504 do_handshake_on_connect=do_handshake_on_connect,
505 suppress_ragged_eofs=suppress_ragged_eofs,
506 server_hostname=server_hostname,
507 context=self,
508 session=session
509 )
File ~\AppData\Local\Programs\Python\Python39\lib\ssl.py:1041, in SSLSocket._create(cls, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, context, session)
1040 raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
-> 1041 self.do_handshake()
1042 except (OSError, ValueError):
File ~\AppData\Local\Programs\Python\Python39\lib\ssl.py:1310, in SSLSocket.do_handshake(self, block)
1309 self.settimeout(None)
-> 1310 self._sslobj.do_handshake()
1311 finally:
ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host
During handling of the above exception, another exception occurred:
ProtocolError Traceback (most recent call last)
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\adapters.py:489, in HTTPAdapter.send(self, request, stream, timeout, verify, cert, proxies)
488 if not chunked:
--> 489 resp = conn.urlopen(
490 method=request.method,
491 url=url,
492 body=request.body,
493 headers=request.headers,
494 redirect=False,
495 assert_same_host=False,
496 preload_content=False,
497 decode_content=False,
498 retries=self.max_retries,
499 timeout=timeout,
500 )
502 # Send the request.
503 else:
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py:787, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
785 e = ProtocolError("Connection aborted.", e)
--> 787 retries = retries.increment(
788 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
789 )
790 retries.sleep()
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\util\retry.py:550, in Retry.increment(self, method, url, response, error, _pool, _stacktrace)
549 if read is False or not self._is_method_retryable(method):
--> 550 raise six.reraise(type(error), error, _stacktrace)
551 elif read is not None:
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\packages\six.py:769, in reraise(tp, value, tb)
768 if value.__traceback__ is not tb:
--> 769 raise value.with_traceback(tb)
770 raise value
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py:703, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
702 # Make the request on the httplib connection object.
--> 703 httplib_response = self._make_request(
704 conn,
705 method,
706 url,
707 timeout=timeout_obj,
708 body=body,
709 headers=headers,
710 chunked=chunked,
711 )
713 # If we're going to release the connection in ``finally:``, then
714 # the response doesn't need to know about the connection. Otherwise
715 # it will also try to release it and we'll have a double-release
716 # mess.
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py:386, in HTTPConnectionPool._make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
385 try:
--> 386 self._validate_conn(conn)
387 except (SocketTimeout, BaseSSLError) as e:
388 # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py:1042, in HTTPSConnectionPool._validate_conn(self, conn)
1041 if not getattr(conn, "sock", None): # AppEngine might not have `.sock`
-> 1042 conn.connect()
1044 if not conn.is_verified:
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connection.py:414, in HTTPSConnection.connect(self)
412 context.load_default_certs()
--> 414 self.sock = ssl_wrap_socket(
415 sock=conn,
416 keyfile=self.key_file,
417 certfile=self.cert_file,
418 key_password=self.key_password,
419 ca_certs=self.ca_certs,
420 ca_cert_dir=self.ca_cert_dir,
421 ca_cert_data=self.ca_cert_data,
422 server_hostname=server_hostname,
423 ssl_context=context,
424 tls_in_tls=tls_in_tls,
425 )
427 # If we're using all defaults and the connection
428 # is TLSv1 or TLSv1.1 we throw a DeprecationWarning
429 # for the host.
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\util\ssl_.py:449, in ssl_wrap_socket(sock, keyfile, certfile, cert_reqs, ca_certs, server_hostname, ssl_version, ciphers, ssl_context, ca_cert_dir, key_password, ca_cert_data, tls_in_tls)
448 if send_sni:
--> 449 ssl_sock = _ssl_wrap_socket_impl(
450 sock, context, tls_in_tls, server_hostname=server_hostname
451 )
452 else:
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\util\ssl_.py:493, in _ssl_wrap_socket_impl(sock, ssl_context, tls_in_tls, server_hostname)
492 if server_hostname:
--> 493 return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
494 else:
File ~\AppData\Local\Programs\Python\Python39\lib\ssl.py:501, in SSLContext.wrap_socket(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)
495 def wrap_socket(self, sock, server_side=False,
496 do_handshake_on_connect=True,
497 suppress_ragged_eofs=True,
498 server_hostname=None, session=None):
499 # SSLSocket class handles server_hostname encoding before it calls
500 # ctx._wrap_socket()
--> 501 return self.sslsocket_class._create(
502 sock=sock,
503 server_side=server_side,
504 do_handshake_on_connect=do_handshake_on_connect,
505 suppress_ragged_eofs=suppress_ragged_eofs,
506 server_hostname=server_hostname,
507 context=self,
508 session=session
509 )
File ~\AppData\Local\Programs\Python\Python39\lib\ssl.py:1041, in SSLSocket._create(cls, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, context, session)
1040 raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
-> 1041 self.do_handshake()
1042 except (OSError, ValueError):
File ~\AppData\Local\Programs\Python\Python39\lib\ssl.py:1310, in SSLSocket.do_handshake(self, block)
1309 self.settimeout(None)
-> 1310 self._sslobj.do_handshake()
1311 finally:
ProtocolError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))
During handling of the above exception, another exception occurred:
ConnectionError Traceback (most recent call last)
Input In [423], in <cell line: 9>()
6 except (ConnectionResetError, ConnectionError):
7 x()
----> 9 x()
Input In [423], in x()
3 mongo_totals = query_db_totals(mongo_db, parameter_map)
4 updates_queue = compare(request_totals, mongo_totals)
----> 5 update_records(updates_queue, mongo_db, headers=headers)
6 except (ConnectionResetError, ConnectionError):
7 x()
Input In [422], in update_records(updates_queue, mongo_db, rate_limit, headers)
41 params = dict(offset=offset, limit=limit)
43 try:
---> 44 response = requests.get(url+collection, headers=headers, params=params).json()
45 except (ConnectionResetError, ConnectionError) as err:
46 request_errors.append(collection)
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\api.py:73, in get(url, params, **kwargs)
62 def get(url, params=None, **kwargs):
63 r"""Sends a GET request.
64
65 :param url: URL for the new :class:`Request` object.
(...)
70 :rtype: requests.Response
71 """
---> 73 return request("get", url, params=params, **kwargs)
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\api.py:59, in request(method, url, **kwargs)
55 # By using the 'with' statement we are sure the session is closed, thus we
56 # avoid leaving sockets open which can trigger a ResourceWarning in some
57 # cases, and look like a memory leak in others.
58 with sessions.Session() as session:
---> 59 return session.request(method=method, url=url, **kwargs)
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\sessions.py:587, in Session.request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
582 send_kwargs = {
583 "timeout": timeout,
584 "allow_redirects": allow_redirects,
585 }
586 send_kwargs.update(settings)
--> 587 resp = self.send(prep, **send_kwargs)
589 return resp
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\sessions.py:701, in Session.send(self, request, **kwargs)
698 start = preferred_clock()
700 # Send the request
--> 701 r = adapter.send(request, **kwargs)
703 # Total elapsed time of the request (approximately)
704 elapsed = preferred_clock() - start
File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\adapters.py:547, in HTTPAdapter.send(self, request, stream, timeout, verify, cert, proxies)
544 raise
546 except (ProtocolError, OSError) as err:
--> 547 raise ConnectionError(err, request=request)
549 except MaxRetryError as e:
550 if isinstance(e.reason, ConnectTimeoutError):
551 # TODO: Remove this in 3.0.0: see #2811
ConnectionError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))
Im using Jupyter Notebook to run a fairly basic crypto trading bot with an API on coinbase and after an arbitrary amount of time, sometimes 6 hours other times 18 hours, the code will error and stop working.. can has anyone had this problem or are able to determine the problem from the error script produced.
error code given below..
OSError Traceback (most recent call last)
~\anaconda3\lib\site-packages\requests\packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
596 # Make the request on the httplib connection object.
--> 597 httplib_response = self._make_request(conn, method, url,
598 timeout=timeout_obj,
~\anaconda3\lib\site-packages\requests\packages\urllib3\connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
385 # otherwise it looks like a programming error was the cause.
--> 386 six.raise_from(e, None)
387 except (SocketTimeout, BaseSSLError, SocketError) as e:
~\anaconda3\lib\site-packages\requests\packages\urllib3\packages\six.py in raise_from(value, from_value)
~\anaconda3\lib\site-packages\requests\packages\urllib3\connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
381 try:
--> 382 httplib_response = conn.getresponse()
383 except Exception as e:
~\anaconda3\lib\http\client.py in getresponse(self)
1370 try:
-> 1371 response.begin()
1372 except ConnectionError:
~\anaconda3\lib\http\client.py in begin(self)
318 while True:
--> 319 version, status, reason = self._read_status()
320 if status != CONTINUE:
~\anaconda3\lib\http\client.py in _read_status(self)
279 def _read_status(self):
--> 280 line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
281 if len(line) > _MAXLINE:
~\anaconda3\lib\socket.py in readinto(self, b)
703 try:
--> 704 return self._sock.recv_into(b)
705 except timeout:
~\anaconda3\lib\site-packages\requests\packages\urllib3\contrib\pyopenssl.py in recv_into(self, *args, **kwargs)
292 else:
--> 293 return self.recv_into(*args, **kwargs)
294
~\anaconda3\lib\site-packages\requests\packages\urllib3\contrib\pyopenssl.py in recv_into(self, *args, **kwargs)
281 else:
--> 282 raise SocketError(str(e))
283 except OpenSSL.SSL.ZeroReturnError as e:
OSError: (10054, 'WSAECONNRESET')
During handling of the above exception, another exception occurred:
ProtocolError Traceback (most recent call last)
~\anaconda3\lib\site-packages\requests\adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
412 if not chunked:
--> 413 resp = conn.urlopen(
414 method=request.method,
~\anaconda3\lib\site-packages\requests\packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
647
--> 648 retries = retries.increment(method, url, error=e, _pool=self,
649 _stacktrace=sys.exc_info()[2])
~\anaconda3\lib\site-packages\requests\packages\urllib3\util\retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
346 if read is False or not self._is_method_retryable(method):
--> 347 raise six.reraise(type(error), error, _stacktrace)
348 elif read is not None:
~\anaconda3\lib\site-packages\requests\packages\urllib3\packages\six.py in reraise(tp, value, tb)
684 if value.__traceback__ is not tb:
--> 685 raise value.with_traceback(tb)
686 raise value
~\anaconda3\lib\site-packages\requests\packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
596 # Make the request on the httplib connection object.
--> 597 httplib_response = self._make_request(conn, method, url,
598 timeout=timeout_obj,
~\anaconda3\lib\site-packages\requests\packages\urllib3\connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
385 # otherwise it looks like a programming error was the cause.
--> 386 six.raise_from(e, None)
387 except (SocketTimeout, BaseSSLError, SocketError) as e:
~\anaconda3\lib\site-packages\requests\packages\urllib3\packages\six.py in raise_from(value, from_value)
~\anaconda3\lib\site-packages\requests\packages\urllib3\connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
381 try:
--> 382 httplib_response = conn.getresponse()
383 except Exception as e:
~\anaconda3\lib\http\client.py in getresponse(self)
1370 try:
-> 1371 response.begin()
1372 except ConnectionError:
~\anaconda3\lib\http\client.py in begin(self)
318 while True:
--> 319 version, status, reason = self._read_status()
320 if status != CONTINUE:
~\anaconda3\lib\http\client.py in _read_status(self)
279 def _read_status(self):
--> 280 line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
281 if len(line) > _MAXLINE:
~\anaconda3\lib\socket.py in readinto(self, b)
703 try:
--> 704 return self._sock.recv_into(b)
705 except timeout:
~\anaconda3\lib\site-packages\requests\packages\urllib3\contrib\pyopenssl.py in recv_into(self, *args, **kwargs)
292 else:
--> 293 return self.recv_into(*args, **kwargs)
294
~\anaconda3\lib\site-packages\requests\packages\urllib3\contrib\pyopenssl.py in recv_into(self, *args, **kwargs)
281 else:
--> 282 raise SocketError(str(e))
283 except OpenSSL.SSL.ZeroReturnError as e:
ProtocolError: ('Connection aborted.', OSError("(10054, 'WSAECONNRESET')"))
During handling of the above exception, another exception occurred:
ConnectionError Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_26992/2760925491.py in <module>
36 # dead zone
37 if (price_diff < buy_diff) and (price_diff > sell_diff) and (wait_time<run_time):
---> 38 price_difference()
39 wait_time = (run_time+180)
40 run_log.logger.info("price difference in dead zone")
~\AppData\Local\Temp/ipykernel_26992/3172399590.py in price_difference()
46 global difference
47 global lastDiff
---> 48 price()
49 historic_price()
50 if current_price < hist_avg:
~\AppData\Local\Temp/ipykernel_26992/3172399590.py in price()
23 global current_ask
24 global last_price
---> 25 var=auth_client.get_product_order_book(trade_pair)
26 bids=var.get('bids')
27 asks=var.get('asks')
~\anaconda3\lib\site-packages\cbpro\public_client.py in get_product_order_book(self, product_id, level)
86 """
87 params = {'level': level}
---> 88 return self._send_message('get',
89 '/products/{}/book'.format(product_id),
90 params=params)
~\anaconda3\lib\site-packages\cbpro\public_client.py in _send_message(self, method, endpoint, params, data)
266 """
267 url = self.url + endpoint
--> 268 r = self.session.request(method, url, params=params, data=data,
269 auth=self.auth, timeout=30)
270 return r.json()
~\anaconda3\lib\site-packages\requests\sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
486 }
487 send_kwargs.update(settings)
--> 488 resp = self.send(prep, **send_kwargs)
489
490 return resp
~\anaconda3\lib\site-packages\requests\sessions.py in send(self, request, **kwargs)
607
608 # Send the request
--> 609 r = adapter.send(request, **kwargs)
610
611 # Total elapsed time of the request (approximately)
~\anaconda3\lib\site-packages\requests\adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
471
472 except (ProtocolError, socket.error) as err:
--> 473 raise ConnectionError(err, request=request)
474
475 except MaxRetryError as e:
ConnectionError: ('Connection aborted.', OSError("(10054, 'WSAECONNRESET')"))
A “connection reset” error usually means that the network is ok but the thing at the other end isn’t, or doesn’t want to talk to you, though with today’s “unclean” networks with proxies and firewalls and layering violations everywhere, there are probably more exotic causes as well.
In any case, I would not try to predict it (which is going to be hopelessly brittle) but deal with it when it happens by catching the exception.
so it might not be the perfect solution but it's going to work for me in this case, have a script simply restart the main script if/when it crashes.
from subprocess import run
from time import sleep
# Path and name to the script you are trying to start
file_path = "CBPRO_BTC_USDC_V3.py"
print('program started')
restart_timer = 60
def start_script():
try:
# Make sure 'python' command is available
run("python "+file_path, check=True)
except:
# Script crashed, lets restart it!
handle_crash()
def handle_crash():
print('Restarting in 60 seconds')
sleep(restart_timer) # Restarts the script after 60 seconds
start_script()
start_script()
I don't know if it crashes because the internet connection. But if you want check if you have a internet cnnection you could ping a server for example 8.8.8.8 (Google DNS Server). If the command returns 0 it was sucesfull if it returns 1 it failed.
import os
response_code = os.system("ping 8.8.8.8 -n 1")
print(response_code)
The following code works fine on a jupyter notebook in a compute instance.
However I need to test this locally on Visual Studio Code
import pandas as pd
import datetime
import csv
import numpy as np
from azureml.core import Workspace
from azureml.core import Dataset, Datastore
from azureml.data.datapath import DataPath
from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient
from azureml.core.authentication import AzureCliAuthentication
# #azure-cosmosdb-table
# from azure.cosmosdb.table.tableservice import TableService
# from azure.cosmosdb.table.models import Entity
#Azure blob storage
cli_auth = AzureCliAuthentication()
ws = Workspace(subscription_id="xx",
resource_group="xx",
workspace_name="xx",
auth=cli_auth)
print("Found workspace {} at location {}".format(ws.name, ws.location))
import os
from azureml.core.authentication import ServicePrincipalAuthentication
svc_pr_password = "xx" #os.environ.get("AZUREML_PASSWORD")
svc_pr = ServicePrincipalAuthentication(
tenant_id="xx",
service_principal_id="xx",
service_principal_password=svc_pr_password)
ws = Workspace(
subscription_id="xx",
resource_group="xx",
workspace_name="xx",
auth=svc_pr
)
print("Found workspace {} at location {}".format(ws.name, ws.location))
However this line:
datastore = Datastore.get(ws, 'yy')
gets me this error:
---------------------------------------------------------------------------
Error Traceback (most recent call last)
~\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\contrib\pyopenssl.py in wrap_socket(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname)
487 try:
--> 488 cnx.do_handshake()
489 except OpenSSL.SSL.WantReadError:
~\AppData\Local\Programs\Python\Python38\lib\site-packages\OpenSSL\SSL.py in do_handshake(self)
1933 result = _lib.SSL_do_handshake(self._ssl)
-> 1934 self._raise_ssl_error(self._ssl, result)
1935
~\AppData\Local\Programs\Python\Python38\lib\site-packages\OpenSSL\SSL.py in _raise_ssl_error(self, ssl, result)
1670 else:
-> 1671 _raise_current_error()
1672
~\AppData\Local\Programs\Python\Python38\lib\site-packages\OpenSSL\_util.py in exception_from_error_queue(exception_type)
53
---> 54 raise exception_type(errors)
55
Error: [('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')]
During handling of the above exception, another exception occurred:
SSLError Traceback (most recent call last)
~\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
669 # Make the request on the httplib connection object.
--> 670 httplib_response = self._make_request(
671 conn,
~\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
380 try:
--> 381 self._validate_conn(conn)
382 except (SocketTimeout, BaseSSLError) as e:
~\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py in _validate_conn(self, conn)
975 if not getattr(conn, "sock", None): # AppEngine might not have `.sock`
--> 976 conn.connect()
977
~\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connection.py in connect(self)
360
--> 361 self.sock = ssl_wrap_socket(
362 sock=conn,
~\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\util\ssl_.py in ssl_wrap_socket(sock, keyfile, certfile, cert_reqs, ca_certs, server_hostname, ssl_version, ciphers, ssl_context, ca_cert_dir, key_password, ca_cert_data)
376 if HAS_SNI and server_hostname is not None:
--> 377 return context.wrap_socket(sock, server_hostname=server_hostname)
378
~\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\contrib\pyopenssl.py in wrap_socket(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname)
493 except OpenSSL.SSL.Error as e:
--> 494 raise ssl.SSLError("bad handshake: %r" % e)
495 break
SSLError: ("bad handshake: Error([('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')])",)
During handling of the above exception, another exception occurred:
MaxRetryError Traceback (most recent call last)
~\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
438 if not chunked:
--> 439 resp = conn.urlopen(
440 method=request.method,
~\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
723
--> 724 retries = retries.increment(
725 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
~\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\util\retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
438 if new_retry.is_exhausted():
--> 439 raise MaxRetryError(_pool, url, error or ResponseError(cause))
440
MaxRetryError: HTTPSConnectionPool(host='westeurope.experiments.azureml.net', port=443): Max retries exceeded with url: /discovery (Caused by SSLError(SSLError("bad handshake: Error([('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')])")))
During handling of the above exception, another exception occurred:
SSLError Traceback (most recent call last)
in
----> 1 datastore = Datastore.get(ws, 'utilisationdb')
~\AppData\Local\Programs\Python\Python38\lib\site-packages\azureml\core\datastore.py in get(workspace, datastore_name)
139 or azureml.data.dbfs_datastore.DBFSDatastore
140 """
--> 141 return Datastore._client().get(workspace, datastore_name)
142
143 #staticmethod
~\AppData\Local\Programs\Python\Python38\lib\site-packages\azureml\data\datastore_client.py in get(workspace, datastore_name)
63 :rtype: AzureFileDatastore or AzureBlobDatastore
64 """
---> 65 return _DatastoreClient._get(workspace, datastore_name)
66
67 #staticmethod
~\AppData\Local\Programs\Python\Python38\lib\site-packages\azureml\data\datastore_client.py in _get(ws, name, auth, host)
541 module_logger.debug("Getting datastore: {}".format(name))
542
--> 543 client = _DatastoreClient._get_client(ws, auth, host)
544 datastore = client.data_stores.get(subscription_id=ws._subscription_id, resource_group_name=ws._resource_group,
545 workspace_name=ws._workspace_name, name=name,
~\AppData\Local\Programs\Python\Python38\lib\site-packages\azureml\data\datastore_client.py in _get_client(ws, auth, host)
726 host_env = os.environ.get('AZUREML_SERVICE_ENDPOINT')
727 auth = auth or ws._auth
--> 728 host = host or host_env or get_service_url(
729 auth, _DatastoreClient._get_workspace_uri_path(ws._subscription_id, ws._resource_group,
730 ws._workspace_name), ws._workspace_id, ws.discovery_url)
~\AppData\Local\Programs\Python\Python38\lib\site-packages\azureml\_base_sdk_common\service_discovery.py in get_service_url(auth, workspace_scope, workspace_id, workspace_discovery_url, service_name)
118
119 cached_service_object = CachedServiceDiscovery(auth)
--> 120 return cached_service_object.get_cached_service_url(workspace_scope, service_name,
121 unique_id=workspace_id, discovery_url=workspace_discovery_url)
122
~\AppData\Local\Programs\Python\Python38\lib\site-packages\azureml\_base_sdk_common\service_discovery.py in get_cached_service_url(self, arm_scope, service_name, unique_id, discovery_url)
280 :rtype: str
281 """
--> 282 return self.get_cached_services_uris(arm_scope, service_name, unique_id=unique_id,
283 discovery_url=discovery_url)[service_name]
~\AppData\Local\Programs\Python\Python38\lib\site-packages\azureml\_base_sdk_common\service_discovery.py in wrapper(self, *args, **kwargs)
180 try:
181 lock_to_use.acquire()
--> 182 return test_function(self, *args, **kwargs)
183 finally:
184 lock_to_use.release()
~\AppData\Local\Programs\Python\Python38\lib\site-packages\azureml\_base_sdk_common\service_discovery.py in get_cached_services_uris(self, arm_scope, service_name, unique_id, discovery_url)
255
256 # Actual service discovery only understands arm_scope
--> 257 cache[cache_key][DEFAULT_FLIGHT] = super(CachedServiceDiscovery, self).discover_services_uris_from_arm_scope(arm_scope, discovery_url)
258 try:
259 with open(self.file_path, "w+") as file:
~\AppData\Local\Programs\Python\Python38\lib\site-packages\azureml\_base_sdk_common\service_discovery.py in discover_services_uris_from_arm_scope(self, arm_scope, discovery_url)
136 def discover_services_uris_from_arm_scope(self, arm_scope, discovery_url=None):
137 discovery_url = self.get_discovery_url(arm_scope, discovery_url)
--> 138 return self.discover_services_uris(discovery_url)
139
140 def discover_services_uris(self, discovery_url=None):
~\AppData\Local\Programs\Python\Python38\lib\site-packages\azureml\_base_sdk_common\service_discovery.py in discover_services_uris(self, discovery_url)
139
140 def discover_services_uris(self, discovery_url=None):
--> 141 status = ClientBase._execute_func(requests.get, discovery_url)
142 status.raise_for_status()
143 return status.json()
~\AppData\Local\Programs\Python\Python38\lib\site-packages\azureml\_restclient\clientbase.py in _execute_func(cls, func, *args, **kwargs)
340 #classmethod
341 def _execute_func(cls, func, *args, **kwargs):
--> 342 return cls._execute_func_internal(
343 DEFAULT_BACKOFF, DEFAULT_RETRIES, module_logger, func, _noop_reset, *args, **kwargs)
344
~\AppData\Local\Programs\Python\Python38\lib\site-packages\azureml\_restclient\clientbase.py in _execute_func_internal(cls, back_off, total_retry, logger, func, reset_func, *args, **kwargs)
334 return func(*args, **kwargs)
335 except Exception as error:
--> 336 left_retry = cls._handle_retry(back_off, left_retry, total_retry, error, logger, func)
337
338 reset_func(*args, **kwargs) # reset_func is expected to undo any side effects from a failed func call.
~\AppData\Local\Programs\Python\Python38\lib\site-packages\azureml\_restclient\clientbase.py in _handle_retry(cls, back_off, left_retry, total_retry, error, logger, func)
364 """
365 if left_retry == 0:
--> 366 raise error
367 elif isinstance(error, HttpOperationError):
368 if error.response.status_code == 403:
~\AppData\Local\Programs\Python\Python38\lib\site-packages\azureml\_restclient\clientbase.py in _execute_func_internal(cls, back_off, total_retry, logger, func, reset_func, *args, **kwargs)
332 while left_retry >= 0:
333 try:
--> 334 return func(*args, **kwargs)
335 except Exception as error:
336 left_retry = cls._handle_retry(back_off, left_retry, total_retry, error, logger, func)
~\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\api.py in get(url, params, **kwargs)
74
75 kwargs.setdefault('allow_redirects', True)
---> 76 return request('get', url, params=params, **kwargs)
77
78
~\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\api.py in request(method, url, **kwargs)
59 # cases, and look like a memory leak in others.
60 with sessions.Session() as session:
---> 61 return session.request(method=method, url=url, **kwargs)
62
63
~\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
528 }
529 send_kwargs.update(settings)
--> 530 resp = self.send(prep, **send_kwargs)
531
532 return resp
~\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\sessions.py in send(self, request, **kwargs)
641
642 # Send the request
--> 643 r = adapter.send(request, **kwargs)
644
645 # Total elapsed time of the request (approximately)
~\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
512 if isinstance(e.reason, _SSLError):
513 # This branch is for urllib3 v1.22 and later.
--> 514 raise SSLError(e, request=request)
515
516 raise ConnectionError(e, request=request)
SSLError: HTTPSConnectionPool(host='westeurope.experiments.azureml.net', port=443): Max retries exceeded with url: /discovery (Caused by SSLError(SSLError("bad handshake: Error([('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')])")))
I want to add coordinates to street names i have downloaded from England's Land Registry but Nonimatim keeps outpouting this error GeocoderUnavailable: Service not available.
I have tried slowing down the requests but still it doesnt help at all.
`from geopy.geocoders import Nominatim
from geopy.distance import vincenty`
geolocator = Nominatim(user_agent="email")
nom=Nominatim(domain='localhost:8080', scheme='http')
`affordable['city_coord'] = affordable['Street'].apply(geolocator.geocode).apply(lambda x: (x.latitude, x.longitude))`
The outpout should be a "city_coord" column with latidue and longitude of every road so i can split it into two later but the actual output is this error:
---------------------------------------------------------------------------
OSError Traceback (most recent call last)
/opt/conda/envs/DSX-Python35/lib/python3.5/urllib/request.py in do_open(self, http_class, req, **http_conn_args)
1253 try:
-> 1254 h.request(req.get_method(), req.selector, req.data, headers)
1255 except OSError as err: # timeout error
/opt/conda/envs/DSX-Python35/lib/python3.5/http/client.py in request(self, method, url, body, headers)
1106 """Send a complete request to the server."""
-> 1107 self._send_request(method, url, body, headers)
1108
/opt/conda/envs/DSX-Python35/lib/python3.5/http/client.py in _send_request(self, method, url, body, headers)
1151 body = _encode(body, 'body')
-> 1152 self.endheaders(body)
1153
/opt/conda/envs/DSX-Python35/lib/python3.5/http/client.py in endheaders(self, message_body)
1102 raise CannotSendHeader()
-> 1103 self._send_output(message_body)
1104
/opt/conda/envs/DSX-Python35/lib/python3.5/http/client.py in _send_output(self, message_body)
933
--> 934 self.send(msg)
935 if message_body is not None:
/opt/conda/envs/DSX-Python35/lib/python3.5/http/client.py in send(self, data)
876 if self.auto_open:
--> 877 self.connect()
878 else:
/opt/conda/envs/DSX-Python35/lib/python3.5/http/client.py in connect(self)
1252
-> 1253 super().connect()
1254
/opt/conda/envs/DSX-Python35/lib/python3.5/http/client.py in connect(self)
848 self.sock = self._create_connection(
--> 849 (self.host,self.port), self.timeout, self.source_address)
850 self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
/opt/conda/envs/DSX-Python35/lib/python3.5/socket.py in create_connection(address, timeout, source_address)
711 if err is not None:
--> 712 raise err
713 else:
/opt/conda/envs/DSX-Python35/lib/python3.5/socket.py in create_connection(address, timeout, source_address)
702 sock.bind(source_address)
--> 703 sock.connect(sa)
704 return sock
OSError: [Errno 101] Network is unreachable
During handling of the above exception, another exception occurred:
URLError Traceback (most recent call last)
/opt/conda/envs/DSX-Python35/lib/python3.5/site-packages/geopy/geocoders/base.py in _call_geocoder(self, url, timeout, raw, requester, deserializer, **kwargs)
354 try:
--> 355 page = requester(req, timeout=timeout, **kwargs)
356 except Exception as error:
/opt/conda/envs/DSX-Python35/lib/python3.5/urllib/request.py in open(self, fullurl, data, timeout)
465
--> 466 response = self._open(req, data)
467
/opt/conda/envs/DSX-Python35/lib/python3.5/urllib/request.py in _open(self, req, data)
483 result = self._call_chain(self.handle_open, protocol, protocol +
--> 484 '_open', req)
485 if result:
/opt/conda/envs/DSX-Python35/lib/python3.5/urllib/request.py in _call_chain(self, chain, kind, meth_name, *args)
443 func = getattr(handler, meth_name)
--> 444 result = func(*args)
445 if result is not None:
/opt/conda/envs/DSX-Python35/lib/python3.5/urllib/request.py in https_open(self, req)
1296 return self.do_open(http.client.HTTPSConnection, req,
-> 1297 context=self._context, check_hostname=self._check_hostname)
1298
/opt/conda/envs/DSX-Python35/lib/python3.5/urllib/request.py in do_open(self, http_class, req, **http_conn_args)
1255 except OSError as err: # timeout error
-> 1256 raise URLError(err)
1257 r = h.getresponse()
URLError: <urlopen error [Errno 101] Network is unreachable>
During handling of the above exception, another exception occurred:
GeocoderUnavailable Traceback (most recent call last)
<ipython-input-14-f2b79a21be8c> in <module>()
----> 1 affordable['city_coord'] = affordable['Street'].apply(geolocator.geocode).apply(lambda x: (x.latitude, x.longitude))
/opt/conda/envs/DSX-Python35/lib/python3.5/site-packages/pandas/core/series.py in apply(self, func, convert_dtype, args, **kwds)
2508 else:
2509 values = self.asobject
-> 2510 mapped = lib.map_infer(values, f, convert=convert_dtype)
2511
2512 if len(mapped) and isinstance(mapped[0], Series):
pandas/_libs/src/inference.pyx in pandas._libs.lib.map_infer()
/opt/conda/envs/DSX-Python35/lib/python3.5/site-packages/geopy/geocoders/osm.py in geocode(self, query, exactly_one, timeout, limit, addressdetails, language, geometry, extratags, country_codes, viewbox, bounded)
385
386 return self._parse_json(
--> 387 self._call_geocoder(url, timeout=timeout), exactly_one
388 )
389
/opt/conda/envs/DSX-Python35/lib/python3.5/site-packages/geopy/geocoders/base.py in _call_geocoder(self, url, timeout, raw, requester, deserializer, **kwargs)
378 raise GeocoderTimedOut('Service timed out')
379 elif "unreachable" in message:
--> 380 raise GeocoderUnavailable('Service not available')
381 elif isinstance(error, SocketTimeout):
382 raise GeocoderTimedOut('Service timed out')
GeocoderUnavailable: Service not available
Try to change apply(geolocator.geocode) to apply(nom.geocode)
affordable['city_coord'] = affordable['Street'].apply(nom.geocode).apply(lambda x: (x.latitude, x.longitude))
I have the following piece of code
import concurrent.futures as cf
from requests_futures.sessions import FuturesSession
urls = ['http://www.foxnews.com/',
'http://www.cnn.com/',
'http://europe.wsj.com/',
'http://www.bbc.co.uk/',
'https://foursquare.com/'] * 500
futures = []
session = FuturesSession(executor=cf.ThreadPoolExecutor(max_workers=8))
for url in urls:
futures.append(session.get(url))
for future in cf.as_completed(futures):
result = len(future.result().content)
print(result)
However this code errors out using a message like
ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
When the length of url list is about 300 it works
but when i have about 500 urls i am back to the same weird error
The complete stack trace is like
---------------------------------------------------------------------------
ConnectionResetError Traceback (most recent call last)
c:\python36\lib\site-packages\requests\packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redi
rect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
599 body=body, headers=headers,
--> 600 chunked=chunked)
601
c:\python36\lib\site-packages\requests\packages\urllib3\connectionpool.py in _make_request(self, conn, method, url, timeout, chunked
, **httplib_request_kw)
385 # otherwise it looks like a programming error was the cause.
--> 386 six.raise_from(e, None)
387 except (SocketTimeout, BaseSSLError, SocketError) as e:
c:\python36\lib\site-packages\requests\packages\urllib3\packages\six.py in raise_from(value, from_value)
c:\python36\lib\site-packages\requests\packages\urllib3\connectionpool.py in _make_request(self, conn, method, url, timeout, chunked
, **httplib_request_kw)
381 try:
--> 382 httplib_response = conn.getresponse()
383 except Exception as e:
c:\python36\lib\http\client.py in getresponse(self)
1330 try:
-> 1331 response.begin()
1332 except ConnectionError:
c:\python36\lib\http\client.py in begin(self)
296 while True:
--> 297 version, status, reason = self._read_status()
298 if status != CONTINUE:
c:\python36\lib\http\client.py in _read_status(self)
257 def _read_status(self):
--> 258 line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
259 if len(line) > _MAXLINE:
c:\python36\lib\socket.py in readinto(self, b)
585 try:
--> 586 return self._sock.recv_into(b)
587 except timeout:
c:\python36\lib\ssl.py in recv_into(self, buffer, nbytes, flags)
1001 self.__class__)
-> 1002 return self.read(nbytes, buffer)
1003 else:
c:\python36\lib\ssl.py in read(self, len, buffer)
864 try:
--> 865 return self._sslobj.read(len, buffer)
866 except SSLError as x:
c:\python36\lib\ssl.py in read(self, len, buffer)
624 if buffer is not None:
--> 625 v = self._sslobj.read(len, buffer)
626 else:
ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host
During handling of the above exception, another exception occurred:
ProtocolError Traceback (most recent call last)
c:\python36\lib\site-packages\requests\adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
422 retries=self.max_retries,
--> 423 timeout=timeout
424 )
c:\python36\lib\site-packages\requests\packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redi
rect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
648 retries = retries.increment(method, url, error=e, _pool=self,
--> 649 _stacktrace=sys.exc_info()[2])
650 retries.sleep()
c:\python36\lib\site-packages\requests\packages\urllib3\util\retry.py in increment(self, method, url, response, error, _pool, _stack
trace)
346 if read is False or not self._is_method_retryable(method):
--> 347 raise six.reraise(type(error), error, _stacktrace)
348 elif read is not None:
c:\python36\lib\site-packages\requests\packages\urllib3\packages\six.py in reraise(tp, value, tb)
684 if value.__traceback__ is not tb:
--> 685 raise value.with_traceback(tb)
686 raise value
c:\python36\lib\site-packages\requests\packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redi
rect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
599 body=body, headers=headers,
--> 600 chunked=chunked)
601
c:\python36\lib\site-packages\requests\packages\urllib3\connectionpool.py in _make_request(self, conn, method, url, timeout, chunked
, **httplib_request_kw)
385 # otherwise it looks like a programming error was the cause.
--> 386 six.raise_from(e, None)
387 except (SocketTimeout, BaseSSLError, SocketError) as e:
c:\python36\lib\site-packages\requests\packages\urllib3\packages\six.py in raise_from(value, from_value)
c:\python36\lib\site-packages\requests\packages\urllib3\connectionpool.py in _make_request(self, conn, method, url, timeout, chunked
, **httplib_request_kw)
381 try:
--> 382 httplib_response = conn.getresponse()
383 except Exception as e:
c:\python36\lib\http\client.py in getresponse(self)
1330 try:
-> 1331 response.begin()
1332 except ConnectionError:
c:\python36\lib\http\client.py in begin(self)
296 while True:
--> 297 version, status, reason = self._read_status()
298 if status != CONTINUE:
c:\python36\lib\http\client.py in _read_status(self)
257 def _read_status(self):
--> 258 line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
259 if len(line) > _MAXLINE:
c:\python36\lib\socket.py in readinto(self, b)
585 try:
--> 586 return self._sock.recv_into(b)
587 except timeout:
c:\python36\lib\ssl.py in recv_into(self, buffer, nbytes, flags)
1001 self.__class__)
-> 1002 return self.read(nbytes, buffer)
1003 else:
c:\python36\lib\ssl.py in read(self, len, buffer)
864 try:
--> 865 return self._sslobj.read(len, buffer)
866 except SSLError as x:
c:\python36\lib\ssl.py in read(self, len, buffer)
624 if buffer is not None:
--> 625 v = self._sslobj.read(len, buffer)
626 else:
ProtocolError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host',
None, 10054, None))
During handling of the above exception, another exception occurred:
ConnectionError Traceback (most recent call last)
<ipython-input-144-e24ea43223c2> in <module>()
15
16 for idx,future in enumerate(cf.as_completed(futures)):
---> 17 result = len(future.result().content)
18 print(idx,result)
19
c:\python36\lib\concurrent\futures\_base.py in result(self, timeout)
396 raise CancelledError()
397 elif self._state == FINISHED:
--> 398 return self.__get_result()
399
400 self._condition.wait(timeout)
c:\python36\lib\concurrent\futures\_base.py in __get_result(self)
355 def __get_result(self):
356 if self._exception:
--> 357 raise self._exception
358 else:
359 return self._result
c:\python36\lib\concurrent\futures\thread.py in run(self)
53
54 try:
---> 55 result = self.fn(*self.args, **self.kwargs)
56 except BaseException as e:
57 self.future.set_exception(e)
c:\python36\lib\site-packages\requests\sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeou
t, allow_redirects, proxies, hooks, stream, verify, cert, json)
486 }
487 send_kwargs.update(settings)
--> 488 resp = self.send(prep, **send_kwargs)
489
490 return resp
c:\python36\lib\site-packages\requests\sessions.py in send(self, request, **kwargs)
607
608 # Send the request
--> 609 r = adapter.send(request, **kwargs)
610
611 # Total elapsed time of the request (approximately)
c:\python36\lib\site-packages\requests\adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
471
472 except (ProtocolError, socket.error) as err:
--> 473 raise ConnectionError(err, request=request)
474
475 except MaxRetryError as e:
ConnectionError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host'
, None, 10054, None))
In [145]:
The problem is with your sites, they limit the number of connections from an IP because of things like DOS attacks (Not an expert).
As you can see in the error ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host