Develop Reference

What is the easiest way to get the url a link redirects to?

I have a number of urls that redirect to other ones and I would like to find the address of the links they redirect to. For example,
https://stlouisfed.org/fomcspeak/thomas-barkin/2019/04/apr-speech-a-practitioners-perspective-on-the-productivity-slowdown
redirects to
https://www.richmondfed.org/press_room/speeches/thomas_i_barkin/2019/barkin_speech_20190404
I tried using the requests library like so:
import requests
url = "https://stlouisfed.org/fomcspeak/thomas-barkin/2019/04/apr-speech-a-practitioners-perspective-on-the-productivity-slowdown"
def get_redirected_url(url: str) -> str:
r = requests.get(url)
return r.url
get_redirected_url(url)
But I'm running getting the error below. Any idea what might be going wrong?
---------------------------------------------------------------------------
Error Traceback (most recent call last)
/opt/anaconda3/lib/python3.8/site-packages/urllib3/contrib/pyopenssl.py in wrap_socket(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname)
487 try:
--> 488 cnx.do_handshake()
489 except OpenSSL.SSL.WantReadError:
/opt/anaconda3/lib/python3.8/site-packages/OpenSSL/SSL.py in do_handshake(self)
1933
-> 1934 :param addr: A remove address
1935 :return: What the socket's connect_ex method returns
/opt/anaconda3/lib/python3.8/site-packages/OpenSSL/SSL.py in _raise_ssl_error(self, ssl, result)
1670
-> 1671 def set_tlsext_host_name(self, name):
1672 """
/opt/anaconda3/lib/python3.8/site-packages/OpenSSL/_util.py in exception_from_error_queue(exception_type)
53 text(lib.ERR_reason_error_string(error)),
---> 54 )
55 )
Error: [('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')]
During handling of the above exception, another exception occurred:
SSLError Traceback (most recent call last)
/opt/anaconda3/lib/python3.8/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
669 # Make the request on the httplib connection object.
--> 670 httplib_response = self._make_request(
671 conn,
/opt/anaconda3/lib/python3.8/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
380 try:
--> 381 self._validate_conn(conn)
382 except (SocketTimeout, BaseSSLError) as e:
/opt/anaconda3/lib/python3.8/site-packages/urllib3/connectionpool.py in _validate_conn(self, conn)
977 if not getattr(conn, "sock", None): # AppEngine might not have `.sock`
--> 978 conn.connect()
979
/opt/anaconda3/lib/python3.8/site-packages/urllib3/connection.py in connect(self)
361
--> 362 self.sock = ssl_wrap_socket(
363 sock=conn,
/opt/anaconda3/lib/python3.8/site-packages/urllib3/util/ssl_.py in ssl_wrap_socket(sock, keyfile, certfile, cert_reqs, ca_certs, server_hostname, ssl_version, ciphers, ssl_context, ca_cert_dir, key_password, ca_cert_data)
385 if HAS_SNI and server_hostname is not None:
--> 386 return context.wrap_socket(sock, server_hostname=server_hostname)
387
/opt/anaconda3/lib/python3.8/site-packages/urllib3/contrib/pyopenssl.py in wrap_socket(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname)
493 except OpenSSL.SSL.Error as e:
--> 494 raise ssl.SSLError("bad handshake: %r" % e)
495 break
SSLError: ("bad handshake: Error([('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')])",)
During handling of the above exception, another exception occurred:
MaxRetryError Traceback (most recent call last)
/opt/anaconda3/lib/python3.8/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
438 if not chunked:
--> 439 resp = conn.urlopen(
440 method=request.method,
/opt/anaconda3/lib/python3.8/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
725
--> 726 retries = retries.increment(
727 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
/opt/anaconda3/lib/python3.8/site-packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
445 if new_retry.is_exhausted():
--> 446 raise MaxRetryError(_pool, url, error or ResponseError(cause))
447
MaxRetryError: HTTPSConnectionPool(host='stlouisfed.org', port=443): Max retries exceeded with url: /fomcspeak/thomas-barkin/2019/04/apr-speech-a-practitioners-perspective-on-the-productivity-slowdown (Caused by SSLError(SSLError("bad handshake: Error([('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')])")))
During handling of the above exception, another exception occurred:
SSLError Traceback (most recent call last)
<ipython-input-126-5a66b7870cb5> in <module>
----> 1 get_redirected_url(url)
<ipython-input-125-d2cc1a52d8fa> in get_redirected_url(url)
1 def get_redirected_url(url: str) -> str:
2 # url = urljoin("https://stlouisfed.org/", url)
----> 3 r = requests.get(url, timeout=10)
4 return r.url
5
/opt/anaconda3/lib/python3.8/site-packages/requests/api.py in get(url, params, **kwargs)
73
74 kwargs.setdefault('allow_redirects', True)
---> 75 return request('get', url, params=params, **kwargs)
76
77
/opt/anaconda3/lib/python3.8/site-packages/requests/api.py in request(method, url, **kwargs)
58 # cases, and look like a memory leak in others.
59 with sessions.Session() as session:
---> 60 return session.request(method=method, url=url, **kwargs)
61
62
/opt/anaconda3/lib/python3.8/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
531 }
532 send_kwargs.update(settings)
--> 533 resp = self.send(prep, **send_kwargs)
534
535 return resp
/opt/anaconda3/lib/python3.8/site-packages/requests/sessions.py in send(self, request, **kwargs)
644
645 # Send the request
--> 646 r = adapter.send(request, **kwargs)
647
648 # Total elapsed time of the request (approximately)
/opt/anaconda3/lib/python3.8/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
512 if isinstance(e.reason, _SSLError):
513 # This branch is for urllib3 v1.22 and later.
--> 514 raise SSLError(e, request=request)
515
516 raise ConnectionError(e, request=request)
SSLError: HTTPSConnectionPool(host='stlouisfed.org', port=443): Max retries exceeded with url: /fomcspeak/thomas-barkin/2019/04/apr-speech-a-practitioners-perspective-on-the-productivity-slowdown (Caused by SSLError(SSLError("bad handshake: Error([('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')])")))

I reproduced your issue with the following code:
import requests
url = "https://stlouisfed.org/fomcspeak/thomas-barkin/2019/04/apr-speech-a-practitioners-perspective-on-the-productivity-slowdown"
def get_redirected_url(url):
r = requests.get(url)
return r.url
get_redirected_url(url)
And the root cause is the server stlouisfed.org doesn't provide the right certificate chain, the L1K intermediate certificate is missing. You can confirm it by checking https://www.ssllabs.com/ssltest/analyze.html?d=https://stlouisfed.org/
To solve/workaround it on your side (client-side), I found 2 options:
you can change url = "https://stlouisfed.org/fomcspeak/thomas-barkin/2019/04/apr-speech-a-practitioners-perspective-on-the-productivity-slowdown"
to
url = "https://www.stlouisfed.org/fomcspeak/thomas-barkin/2019/04/apr-speech-a-practitioners-perspective-on-the-productivity-slowdown"
as the www.stlouisfed.org certificate chain is valid.
you can add the missing certificate L1K into your certificate trust file:
first, download the certificate from
https://tls-observatory.services.mozilla.com/static/certsplainer.html?id=1439
2nd, update your certificate trust file, in my centos 7, I need to
add a new file called l1k.pem into /etc/pki/ca-trust/source/anchors with the content mentioned in step1, and then run command update-ca-trust; then confirm the certificate is added in file /etc/pki/tls/certs/ca-bundle.crt.
With either option, the redirected URL can be returned.

Spotify Python API call timeout issues

I am getting below error on Spotify API call. is there work around to resolve these timeout issues.
Python Spotify Code
TimeoutError Traceback (most recent call last)
C:\ProgramData\Anaconda3\lib\site-packages\urllib3\util\connection.py in create_connection(address, timeout, source_address, socket_options)
73 sock.bind(source_address)
---> 74 sock.connect(sa)
75 return sock
TimeoutError: [WinError 10060] A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond
During handling of the above exception, another exception occurred:
KeyboardInterrupt Traceback (most recent call last)
<ipython-input-11-eb4cef5ffdbb> in <module>
5 #print(artist)
6 #time.sleep(2)
----> 7 result = spotify.search(artist,search_type='artist')['artists']['items']
8 #print(result)
9 try:
<ipython-input-7-6a2d183a2f3a> in search(self, query, search_type)
80 data = urlencode({"q": query, "type": search_type.lower()})
81 lookup_url = f"{endpoint}?{data}"
---> 82 r = requests.get(lookup_url, headers=headers)#.json()
83 if r.status_code not in range(200, 299):
84 return {}
C:\ProgramData\Anaconda3\lib\site-packages\requests\api.py in get(url, params, **kwargs)
74
75 kwargs.setdefault('allow_redirects', True)
---> 76 return request('get', url, params=params, **kwargs)
77
78
C:\ProgramData\Anaconda3\lib\site-packages\requests\api.py in request(method, url, **kwargs)
59 # cases, and look like a memory leak in others.
60 with sessions.Session() as session:
---> 61 return session.request(method=method, url=url, **kwargs)
62
63
C:\ProgramData\Anaconda3\lib\site-packages\requests\sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
528 }
529 send_kwargs.update(settings)
--> 530 resp = self.send(prep, **send_kwargs)
531
532 return resp
C:\ProgramData\Anaconda3\lib\site-packages\requests\sessions.py in send(self, request, **kwargs)
641
642 # Send the request
--> 643 r = adapter.send(request, **kwargs)
644
645 # Total elapsed time of the request (approximately)
C:\ProgramData\Anaconda3\lib\site-packages\requests\adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
437 try:
438 if not chunked:
--> 439 resp = conn.urlopen(
440 method=request.method,
441 url=url,
C:\ProgramData\Anaconda3\lib\site-packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
668
669 # Make the request on the httplib connection object.
--> 670 httplib_response = self._make_request(
671 conn,
672 method,
C:\ProgramData\Anaconda3\lib\site-packages\urllib3\connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
379 # Trigger any extra validation we need to do.
380 try:
--> 381 self._validate_conn(conn)
382 except (SocketTimeout, BaseSSLError) as e:
383 # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.
C:\ProgramData\Anaconda3\lib\site-packages\urllib3\connectionpool.py in _validate_conn(self, conn)
974 # Force connect early to allow us to validate the connection.
975 if not getattr(conn, "sock", None): # AppEngine might not have `.sock`
--> 976 conn.connect()
977
978 if not conn.is_verified:
C:\ProgramData\Anaconda3\lib\site-packages\urllib3\connection.py in connect(self)
306 def connect(self):
307 # Add certificate verification
--> 308 conn = self._new_conn()
309 hostname = self.host
310
C:\ProgramData\Anaconda3\lib\site-packages\urllib3\connection.py in _new_conn(self)
157
158 try:
--> 159 conn = connection.create_connection(
160 (self._dns_host, self.port), self.timeout, **extra_kw
161 )
C:\ProgramData\Anaconda3\lib\site-packages\urllib3\util\connection.py in create_connection(address, timeout, source_address, socket_options)
72 if source_address:
73 sock.bind(source_address)
---> 74 sock.connect(sa)
75 return sock
76
KeyboardInterrupt:
1
###########Create Data frame with required columns to be extracte
[1]: https://i.stack.imgur.com/lCaz4.p
ng

I really wish spotipy would address this issue in their package, but nonetheless, here's my workaround:
Import the error type
If you don't catch this exact error type, you'll cause problems for yourself by leaving your catch open-ended. For example, if you have a TypeError in your syntax or something, it will error that out and go to your catch block, and you'll have no way of knowing that it errored in the first place.
from requests.exceptions import ReadTimeout
When you init your spotipy client, I recommend adding in a couple of params that might be helpful.
spotify = spotipy.Spotify(auth=spotify_token, requests_timeout=10, retries=10)
requests_timeout | throws the same error you're looking for after a number of seconds. I use this just so that it doesn't hang forever, and it'll just retry the request if nothing comes back after 10 seconds.
retries | the maximum number of retries spotipy will request if it gets an error. I think it defaults to 3, so I just like to increase this a little bit just in case.
Run a try/except type retry system
You can definitely expand this to make it keep retrying (using a while loop or something) if it keeps hanging, but this is the basic idea:
try:
result = spotify.search(artist,search_type='artist')['artists']['items']
except ReadTimeout:
print('Spotify timed out... trying again...')
result = spotify.search(artist,search_type='artist')['artists']['items']

cannot import texthero in python

I am trying to import and use the texthero module in python and keep getting an error message.
code is simply import texthero as hero
there is a really long error message
[nltk_data] Error loading stopwords: <urlopen error [WinError 10060] A
[nltk_data] connection attempt failed because the connected party
[nltk_data] did not properly respond after a period of time, or
[nltk_data] established connection failed because connected host
[nltk_data] has failed to respond>
---------------------------------------------------------------------------
OSError Traceback (most recent call last)
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\texthero\stopwords.py in <module>()
13 # If not present, download 'en_core_web_sm'
---> 14 spacy_model = spacy.load("en_core_web_sm")
15 except OSError:
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\spacy\__init__.py in load(name, **overrides)
29 warnings.warn(Warnings.W001.format(path=depr_path), DeprecationWarning)
---> 30 return util.load_model(name, **overrides)
31
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\spacy\util.py in load_model(name, **overrides)
174 return load_model_from_path(name, **overrides)
--> 175 raise IOError(Errors.E050.format(name=name))
176
OSError: [E050] Can't find model 'en_core_web_sm'. It doesn't seem to be a shortcut link, a Python package or a valid path to a data directory.
During handling of the above exception, another exception occurred:
TimeoutError Traceback (most recent call last)
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\urllib3\connection.py in _new_conn(self)
159 conn = connection.create_connection(
--> 160 (self._dns_host, self.port), self.timeout, **extra_kw
161 )
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\urllib3\util\connection.py in create_connection(address, timeout, source_address, socket_options)
83 if err is not None:
---> 84 raise err
85
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\urllib3\util\connection.py in create_connection(address, timeout, source_address, socket_options)
73 sock.bind(source_address)
---> 74 sock.connect(sa)
75 return sock
TimeoutError: [WinError 10060] A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond
During handling of the above exception, another exception occurred:
NewConnectionError Traceback (most recent call last)
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
676 headers=headers,
--> 677 chunked=chunked,
678 )
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\urllib3\connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
380 try:
--> 381 self._validate_conn(conn)
382 except (SocketTimeout, BaseSSLError) as e:
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\urllib3\connectionpool.py in _validate_conn(self, conn)
977 if not getattr(conn, "sock", None): # AppEngine might not have `.sock`
--> 978 conn.connect()
979
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\urllib3\connection.py in connect(self)
308 # Add certificate verification
--> 309 conn = self._new_conn()
310 hostname = self.host
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\urllib3\connection.py in _new_conn(self)
171 raise NewConnectionError(
--> 172 self, "Failed to establish a new connection: %s" % e
173 )
NewConnectionError: <urllib3.connection.HTTPSConnection object at 0x0000000019860160>: Failed to establish a new connection: [WinError 10060] A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond
During handling of the above exception, another exception occurred:
MaxRetryError Traceback (most recent call last)
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\requests\adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
448 retries=self.max_retries,
--> 449 timeout=timeout
450 )
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
726 retries = retries.increment(
--> 727 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
728 )
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\urllib3\util\retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
438 if new_retry.is_exhausted():
--> 439 raise MaxRetryError(_pool, url, error or ResponseError(cause))
440
MaxRetryError: HTTPSConnectionPool(host='raw.githubusercontent.com', port=443): Max retries exceeded with url: /explosion/spacy-models/master/shortcuts-v2.json (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x0000000019860160>: Failed to establish a new connection: [WinError 10060] A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond',))
During handling of the above exception, another exception occurred:
ConnectionError Traceback (most recent call last)
<ipython-input-3-a19be4912cd0> in <module>()
----> 1 import texthero as hero
2
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\texthero\__init__.py in <module>()
4
5 """
----> 6 from . import preprocessing
7 from .preprocessing import *
8
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\texthero\preprocessing.py in <module>()
12 from nltk.stem import PorterStemmer, SnowballStemmer
13
---> 14 from texthero import stopwords as _stopwords
15
16 from typing import List, Callable
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\texthero\stopwords.py in <module>()
16 from spacy.cli.download import download as spacy_download
17
---> 18 spacy_download("en_core_web_sm")
19
20 from spacy.lang.en import stop_words as spacy_en_stopwords
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\spacy\cli\download.py in download(model, direct, *pip_args)
42 dl = download_model(dl_tpl.format(m=model_name, v=version), pip_args)
43 else:
---> 44 shortcuts = get_json(about.__shortcuts__, "available shortcuts")
45 model_name = shortcuts.get(model, model)
46 compatibility = get_compatibility()
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\spacy\cli\download.py in get_json(url, desc)
93
94 def get_json(url, desc):
---> 95 r = requests.get(url)
96 if r.status_code != 200:
97 msg.fail(
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\requests\api.py in get(url, params, **kwargs)
74
75 kwargs.setdefault('allow_redirects', True)
---> 76 return request('get', url, params=params, **kwargs)
77
78
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\requests\api.py in request(method, url, **kwargs)
59 # cases, and look like a memory leak in others.
60 with sessions.Session() as session:
---> 61 return session.request(method=method, url=url, **kwargs)
62
63
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\requests\sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
528 }
529 send_kwargs.update(settings)
--> 530 resp = self.send(prep, **send_kwargs)
531
532 return resp
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\requests\sessions.py in send(self, request, **kwargs)
641
642 # Send the request
--> 643 r = adapter.send(request, **kwargs)
644
645 # Total elapsed time of the request (approximately)
C:\Users\name\AppData\Roaming\Python\Python36\site-packages\requests\adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
514 raise SSLError(e, request=request)
515
--> 516 raise ConnectionError(e, request=request)
517
518 except ClosedPoolError as e:
ConnectionError: HTTPSConnectionPool(host='raw.githubusercontent.com', port=443): Max retries exceeded with url: /explosion/spacy-models/master/shortcuts-v2.json (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x0000000019860160>: Failed to establish a new connection: [WinError 10060] A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond',))
So it seems not to be able to open the stopwords file. I have checked the data folders and it is present.
when it can't find it it then tries to find the spacy file 'en_core_web_sm'
I cannot find this in the folders although spacy is there.
I then get a time out as it can't find the file.
I have tried uninstalling and reinstalling spacy and texthero and updating both.
I have tired installing en_core_web_sm and the other versions
e.g. using code:
!pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.2.0/en_core_web_sm-2.2.0.tar.gz
and
!python -m spacy download en_core_web_sm
I have tried reinstalling/updating nltk
I have run in Jupyter notebook and in Python and CMD (as admin). all with the same error.
I've tried both python commands and pip to do the above installs etc.
I really don't know what else to try
any ideas?
thanks
Mizz

How to select links from a list of links which are giving some kind of error?

I have a list of links. Many of them are giving errors like 404: page note found and many are not getting connected ie timeout error. They might give all different kinds of error, I have not checked all. I want to separate good links which are working and bad links which give any kind of error.
I tried to filter out the connection timed out links like this:
import requests
r=requests.get("http://www.carillionplc.co.uk/sustain/f_con2.htm", timeout=10)
But I got the following error:
---------------------------------------------------------------------------
timeout Traceback (most recent call last)
~/anaconda3/lib/python3.7/site-packages/urllib3/connection.py in _new_conn(self)
158 conn = connection.create_connection(
--> 159 (self._dns_host, self.port), self.timeout, **extra_kw)
160
~/anaconda3/lib/python3.7/site-packages/urllib3/util/connection.py in create_connection(address, timeout, source_address, socket_options)
79 if err is not None:
---> 80 raise err
81
~/anaconda3/lib/python3.7/site-packages/urllib3/util/connection.py in create_connection(address, timeout, source_address, socket_options)
69 sock.bind(source_address)
---> 70 sock.connect(sa)
71 return sock
timeout: timed out
During handling of the above exception, another exception occurred:
ConnectTimeoutError Traceback (most recent call last)
~/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
599 body=body, headers=headers,
--> 600 chunked=chunked)
601
~/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
353 else:
--> 354 conn.request(method, url, **httplib_request_kw)
355
~/anaconda3/lib/python3.7/http/client.py in request(self, method, url, body, headers, encode_chunked)
1228 """Send a complete request to the server."""
-> 1229 self._send_request(method, url, body, headers, encode_chunked)
1230
~/anaconda3/lib/python3.7/http/client.py in _send_request(self, method, url, body, headers, encode_chunked)
1274 body = _encode(body, 'body')
-> 1275 self.endheaders(body, encode_chunked=encode_chunked)
1276
~/anaconda3/lib/python3.7/http/client.py in endheaders(self, message_body, encode_chunked)
1223 raise CannotSendHeader()
-> 1224 self._send_output(message_body, encode_chunked=encode_chunked)
1225
~/anaconda3/lib/python3.7/http/client.py in _send_output(self, message_body, encode_chunked)
1015 del self._buffer[:]
-> 1016 self.send(msg)
1017
~/anaconda3/lib/python3.7/http/client.py in send(self, data)
955 if self.auto_open:
--> 956 self.connect()
957 else:
~/anaconda3/lib/python3.7/site-packages/urllib3/connection.py in connect(self)
180 def connect(self):
--> 181 conn = self._new_conn()
182 self._prepare_conn(conn)
~/anaconda3/lib/python3.7/site-packages/urllib3/connection.py in _new_conn(self)
163 self, "Connection to %s timed out. (connect timeout=%s)" %
--> 164 (self.host, self.timeout))
165
ConnectTimeoutError: (<urllib3.connection.HTTPConnection object at 0x7f7c86c14ba8>, 'Connection to www.carillionplc.co.uk timed out. (connect timeout=10)')
During handling of the above exception, another exception occurred:
MaxRetryError Traceback (most recent call last)
~/anaconda3/lib/python3.7/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
448 retries=self.max_retries,
--> 449 timeout=timeout
450 )
~/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
637 retries = retries.increment(method, url, error=e, _pool=self,
--> 638 _stacktrace=sys.exc_info()[2])
639 retries.sleep()
~/anaconda3/lib/python3.7/site-packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
398 if new_retry.is_exhausted():
--> 399 raise MaxRetryError(_pool, url, error or ResponseError(cause))
400
MaxRetryError: HTTPConnectionPool(host='www.carillionplc.co.uk', port=80): Max retries exceeded with url: /sustain/f_con2.htm (Caused by ConnectTimeoutError(<urllib3.connection.HTTPConnection object at 0x7f7c86c14ba8>, 'Connection to www.carillionplc.co.uk timed out. (connect timeout=10)'))
During handling of the above exception, another exception occurred:
ConnectTimeout Traceback (most recent call last)
<ipython-input-6-425ceeca52ad> in <module>
1 import requests
----> 2 r=requests.get("http://www.carillionplc.co.uk/sustain/f_con2.htm", timeout=10)
~/anaconda3/lib/python3.7/site-packages/requests/api.py in get(url, params, **kwargs)
73
74 kwargs.setdefault('allow_redirects', True)
---> 75 return request('get', url, params=params, **kwargs)
76
77
~/anaconda3/lib/python3.7/site-packages/requests/api.py in request(method, url, **kwargs)
58 # cases, and look like a memory leak in others.
59 with sessions.Session() as session:
---> 60 return session.request(method=method, url=url, **kwargs)
61
62
~/anaconda3/lib/python3.7/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
531 }
532 send_kwargs.update(settings)
--> 533 resp = self.send(prep, **send_kwargs)
534
535 return resp
~/anaconda3/lib/python3.7/site-packages/requests/sessions.py in send(self, request, **kwargs)
644
645 # Send the request
--> 646 r = adapter.send(request, **kwargs)
647
648 # Total elapsed time of the request (approximately)
~/anaconda3/lib/python3.7/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
502 # TODO: Remove this in 3.0.0: see #2811
503 if not isinstance(e.reason, NewConnectionError):
--> 504 raise ConnectTimeout(e, request=request)
505
506 if isinstance(e.reason, ResponseError):
ConnectTimeout: HTTPConnectionPool(host='www.carillionplc.co.uk', port=80): Max retries exceeded with url: /sustain/f_con2.htm (Caused by ConnectTimeoutError(<urllib3.connection.HTTPConnection object at 0x7f7c86c14ba8>, 'Connection to www.carillionplc.co.uk timed out. (connect timeout=10)'))
Is there a simple way to just filter out the working links.

You can't know which links are working unless you try them.
So, if you required to filter broken links you can try this:
import requests
links = [...] #your links are stored here
working_links = [] #empty list
for link in links:
try:
r=requests.get(link, timeout=10)
working_links.append(link)
except Exception as e:
print(f'request to {link} failed with {str(e)}')

I think you can pass the requests into try/except statement, and remove the ones that reach the except statement from your list..
It would look something like the following:
for link in your_list:
try:
r = requests.get(link, timeout=1)
except:
your_list.remove(link)

Rest service working in Postman but not in Python IDE (Anaconda)

I am trying to invoke a rest service through Jupyter. It is failing with "Tunnel connection failed: 404 Not found". But the same works with Postman.
Please let me know how to handle this scenario in Jupyter.
Following things are configured as environment variables.
HTTP_Proxy = :
HTTPS_Proxy = :
Following is my rest call.
import requests
url = "https://<Url>"
payload = "{\"start\":0,\"end\":1461927918213,\"tags\":[{\"name\":\"OfflineObjects.Demo.Dynamic.Sambit.Float\"}]}"
headers = {
'content-type': "application/json",
'zone-id': "a698eb6c-fc46-452c-a000-50290ba5bf84",
'authorization': "bearer <token>",
'cache-control': "no-cache",
'postman-token': "1fe11475-8017-bbb0-8b19-beb1eadafc60"
}
response = requests.request("POST", url, data=payload, headers=headers)
print(response.text)
I am getting the following exception.
C:\Anaconda3\lib\site-packages\requests\packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, **response_kw)
553 if is_new_proxy_conn:
--> 554 self._prepare_proxy(conn)
555
C:\Anaconda3\lib\site-packages\requests\packages\urllib3\connectionpool.py in _prepare_proxy(self, conn)
749
--> 750 conn.connect()
751
C:\Anaconda3\lib\site-packages\requests\packages\urllib3\connection.py in connect(self)
229 # self._tunnel_host below.
--> 230 self._tunnel()
231 # Mark this connection as not reusable
C:\Anaconda3\lib\http\client.py in _tunnel(self)
808 raise OSError("Tunnel connection failed: %d %s" % (code,
--> 809 message.strip()))
810 while True:
OSError: Tunnel connection failed: 404 Not found
During handling of the above exception, another exception occurred:
MaxRetryError Traceback (most recent call last)
C:\Anaconda3\lib\site-packages\requests\adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
375 retries=self.max_retries,
--> 376 timeout=timeout
377 )
C:\Anaconda3\lib\site-packages\requests\packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, **response_kw)
608 retries = retries.increment(method, url, error=e, _pool=self,
--> 609 _stacktrace=sys.exc_info()[2])
610 retries.sleep()
C:\Anaconda3\lib\site-packages\requests\packages\urllib3\util\retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
272 if new_retry.is_exhausted():
--> 273 raise MaxRetryError(_pool, url, error or ResponseError(cause))
274
MaxRetryError: HTTPSConnectionPool(host='url', port=443): Max retries exceeded with url: /v1/datapoints (Caused by ProxyError('Cannot connect to proxy.', OSError('Tunnel connection failed: 404 Not found',)))
Thanks.