I'm running a script that communicates with a Rest API. Occasionally it's returning an exception that also returns During handling of the above exception, another exception occurred: for two additional exceptions. I'm trying to handle it using try/except blocks, but despite importing all three exceptions and coding for them in the blocks the script still fails.
The three exceptions are http.client.RemoteDisconnected, urllib3.exceptions.ProtocolError and requests.exceptions.ConnectionError.
So my import code is:
from requests.exceptions import ConnectionError
from http.client import RemoteDisconnected
from urllib3.exceptions import ProtocolError
My IDE recognizes the exceptions as imported, so that's probably fine. My except code is:
try
[the thing]
except (InternalError, ConnectionError, RemoteDisconnected, ProtocolError) as e:
self.lg.log(str(e) + ' ' + str(e.__traceback__) + " Some sort of connection or
internal error occured")
I have another exception on the same block (InternalError) and that one seems to handle fine.
The Traceback is really long but I'm not sure which parts are most important, so here's the whole thing
Traceback (most recent call last):
File "C:\Users\Rominus\PycharmProjects\Binance\venv\lib\site-packages\urllib3\connectionpool.py", line 600, in urlopen
chunked=chunked)
File "C:\Users\Rominus\PycharmProjects\Binance\venv\lib\site-packages\urllib3\connectionpool.py", line 384, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "C:\Users\Rominus\PycharmProjects\Binance\venv\lib\site-packages\urllib3\connectionpool.py", line 380, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\Rominus\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 1331, in getresponse
response.begin()
File "C:\Users\Rominus\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 297, in begin
version, status, reason = self._read_status()
File "C:\Users\Rominus\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 266, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Rominus\PycharmProjects\Binance\venv\lib\site-packages\requests\adapters.py", line 445, in send
timeout=timeout
File "C:\Users\Rominus\PycharmProjects\Binance\venv\lib\site-packages\urllib3\connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "C:\Users\Rominus\PycharmProjects\Binance\venv\lib\site-packages\urllib3\util\retry.py", line 367, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Users\Rominus\PycharmProjects\Binance\venv\lib\site-packages\urllib3\packages\six.py", line 685, in reraise
raise value.with_traceback(tb)
File "C:\Users\Rominus\PycharmProjects\Binance\venv\lib\site-packages\urllib3\connectionpool.py", line 600, in urlopen
chunked=chunked)
File "C:\Users\Rominus\PycharmProjects\Binance\venv\lib\site-packages\urllib3\connectionpool.py", line 384, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "C:\Users\Rominus\PycharmProjects\Binance\venv\lib\site-packages\urllib3\connectionpool.py", line 380, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\Rominus\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 1331, in getresponse
response.begin()
File "C:\Users\Rominus\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 297, in begin
version, status, reason = self._read_status()
File "C:\Users\Rominus\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 266, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:/Users/Rominus/PycharmProjects/Binance/Skitter_trade_test.py", line 55, in <module>
bintime = int(client.time()['serverTime'])
File "C:\Users\Rominus\PycharmProjects\Binance\api.py", line 98, in time
return self._get('v1/time')
File "C:\Users\Rominus\PycharmProjects\Binance\api.py", line 62, in _get
return self._request(endpoint, param, "get")
File "C:\Users\Rominus\PycharmProjects\Binance\api.py", line 40, in _request
r = getattr(self.session, http_method)(BinanceAPI.API_URL + endpoint, params=params)
File "C:\Users\Rominus\PycharmProjects\Binance\venv\lib\site-packages\requests\sessions.py", line 525, in get
return self.request('GET', url, **kwargs)
File "C:\Users\Rominus\PycharmProjects\Binance\venv\lib\site-packages\requests\sessions.py", line 512, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\Rominus\PycharmProjects\Binance\venv\lib\site-packages\requests\sessions.py", line 622, in send
r = adapter.send(request, **kwargs)
File "C:\Users\Rominus\PycharmProjects\Binance\venv\lib\site-packages\requests\adapters.py", line 495, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
Related
I'm trying to make automated crypto monitoring program with python and using ccxt library.
(python version : 3.11)
After running this program for 6 to 7 hours, the following error occurred and the program was terminated.
Traceback (most recent call last):
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\connectionpool.py", line 703, in urlopen
httplib_response = self._make_request(
^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\connectionpool.py", line 449, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\connectionpool.py", line 444, in _make_request
httplib_response = conn.getresponse()
^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\http\client.py", line 1374, in getresponse
response.begin()
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\http\client.py", line 318, in begin
version, status, reason = self._read_status()
^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\http\client.py", line 287, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\requests\adapters.py", line 489, in send
resp = conn.urlopen(
^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\connectionpool.py", line 787, in urlopen
retries = retries.increment(
^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\util\retry.py", line 550, in increment
raise six.reraise(type(error), error, _stacktrace)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\packages\six.py", line 769, in reraise
raise value.with_traceback(tb)
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\connectionpool.py", line 703, in urlopen
httplib_response = self._make_request(
^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\connectionpool.py", line 449, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\connectionpool.py", line 444, in _make_request
httplib_response = conn.getresponse()
^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\http\client.py", line 1374, in getresponse
response.begin()
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\http\client.py", line 318, in begin
version, status, reason = self._read_status()
^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\http\client.py", line 287, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\ccxt\base\exchange.py", line 599, in fetch
response = self.session.request(
^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\requests\sessions.py", line 587, in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\requests\sessions.py", line 701, in send
r = adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\requests\adapters.py", line 547, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "c:\Users\uid\Python\PJT_Crypto\Binance_Crypto_realtime.py", line 312, in <module>
Tracking_Crypto_Start_Start('15m', f, f2)
File "c:\Users\uid\Python\PJT_Crypto\Binance_Crypto_realtime.py", line 125, in Tracking_Crypto_Start_Start
get_data = bollinger_trade_logic(exchange, interval)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "c:\Users\uid\Python\PJT_Crypto\Binance_Crypto_realtime.py", line 29, in bollinger_trade_logic
symbol_df = get_data_frame(obj_binance, interval)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "c:\Users\uid\Python\PJT_Crypto\Binance_Crypto_realtime.py", line 19, in get_data_frame
p = obj_binance.fetch_ohlcv(
^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\ccxt\binance.py", line 2709, in fetch_ohlcv
response = getattr(self, method)(self.extend(request, params))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\ccxt\base\exchange.py", line 505, in inner
return entry(_self, **inner_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\ccxt\binance.py", line 6200, in request
response = self.fetch2(path, api, method, params, headers, body, config, context)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\ccxt\base\exchange.py", line 2830, in fetch2
return self.fetch(request['url'], request['method'], request['headers'], request['body'])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\ccxt\base\exchange.py", line 652, in fetch
raise NetworkError(details) from e
ccxt.base.errors.NetworkError: binance GET https://fapi.binance.com/fapi/v1/klines?interval=15m&limit=20&symbol=XRPUSDT
I think the Binance server forcibly rejected the request because I made too many requests.
Main code like this:
def get_data_frame(obj_binance, interval):
p = obj_binance.fetch_ohlcv(
'XRP/USDT', interval, limit=20)
df = pd.DataFrame(p, columns=['date', 'open',
'high', 'low', 'close', 'vol'])
return df
def Start_Monitoring(Param1,Param2):
try:
get_data = get_data_frame(Param1,Param2)
.....
...
time.sleep(1)
except ccxt.RequestTimeout as e:
print(e)
exchange = binanceperps(param) # This line for re-connect & request again. But never catched e untill now
There were many similar articles on the Internet, but I couldn't find a case that fits my case.
Please give me some advice.
Thanks
BR
I tried to add catch/exception code in while loop. But It couldn't capture the error.
The error what I post was occurred in python library not ccxt library.
I receive the following error every few hours. Why?
I think this is related to Stripe payment.
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 665, in urlopen
httplib_response = self._make_request(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 421, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 416, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.8/http/client.py", line 1348, in getresponse
response.begin()
File "/usr/lib/python3.8/http/client.py", line 316, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.8/http/client.py", line 285, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above error some other errors occur:
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/digital/.local/lib/python3.8/site-packages/requests/adapters.py", line 439, in send
resp = conn.urlopen(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 719, in urlopen
retries = retries.increment(
File "/usr/lib/python3/dist-packages/urllib3/util/retry.py", line 400, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/digital/.local/lib/python3.8/site-packages/six.py", line 702, in reraise
raise value.with_traceback(tb)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 665, in urlopen
httplib_response = self._make_request(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 421, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 416, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.8/http/client.py", line 1348, in getresponse
response.begin()
File "/usr/lib/python3.8/http/client.py", line 316, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.8/http/client.py", line 285, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/digital/.local/lib/python3.8/site-packages/stripe/http_client.py", line 286, in request
result = self._thread_local.session.request(
File "/home/digital/.local/lib/python3.8/site-packages/requests/sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "/home/digital/.local/lib/python3.8/site-packages/requests/sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "/home/digital/.local/lib/python3.8/site-packages/requests/adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
self.run()
File "/usr/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/home/digital/paymentfollower.py", line 145, in paymentfollowing
newer_charge=stripe.Charge.list(limit=1,ending_before=older_charge_data)
File "/home/digital/.local/lib/python3.8/site-packages/stripe/api_resources/abstract/listable_api_resource.py", line 23, in list
response, api_key = requestor.request("get", url, params)
File "/home/digital/.local/lib/python3.8/site-packages/stripe/api_requestor.py", line 119, in request
rbody, rcode, rheaders, my_api_key = self.request_raw(
File "/home/digital/.local/lib/python3.8/site-packages/stripe/api_requestor.py", line 343, in request_raw
rbody, rcode, rheaders = self._client.request_with_retries(
File "/home/digital/.local/lib/python3.8/site-packages/stripe/http_client.py", line 151, in request_with_retries
raise connection_error
File "/home/digital/.local/lib/python3.8/site-packages/stripe/http_client.py", line 123, in request_with_retries
response = self.request(method, url, headers, post_data)
File "/home/digital/.local/lib/python3.8/site-packages/stripe/http_client.py", line 312, in request
self._handle_request_error(e)
File "/home/digital/.local/lib/python3.8/site-packages/stripe/http_client.py", line 364, in _handle_request_error
raise error.APIConnectionError(msg, should_retry=should_retry)
stripe.error.APIConnectionError: Unexpected error communicating with Stripe. If this problem persists,
let us know at support#stripe.com.
(Network error: ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')))
Stack-overflow keeps saying that my post is mostly code and I should add more explanation. So I keep repeating this sentence.
Stack-overflow keeps saying that my post is mostly code and I should add more explanation. So I keep repeating this sentence.
stripe.max_network_retries = 2 fixed the issue for me, as was suggested in the comments.
my Code runs on my local network(using VBOX) but when im trying to run it on another network im getting this Error:
docker.errors.DockerException: Error while fetching server API version: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
my code is:
import docker
client = docker.DockerClient(base_url ='ssh://george#192.168.56.103')
print(client.images.list())
the whole Error message is:
C:\Users\George\PycharmProjects\docker_project\venv\Scripts\python.exe C:/Users/George/PycharmProjects/docker_project/docker_test.py
Traceback (most recent call last):
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\urllib3\connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\urllib3\connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\urllib3\connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\George\AppData\Local\Programs\Python\Python38-32\lib\http\client.py", line 1332, in getresponse
response.begin()
File "C:\Users\George\AppData\Local\Programs\Python\Python38-32\lib\http\client.py", line 303, in begin
version, status, reason = self._read_status()
File "C:\Users\George\AppData\Local\Programs\Python\Python38-32\lib\http\client.py", line 272, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\urllib3\connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\urllib3\util\retry.py", line 531, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\urllib3\packages\six.py", line 734, in reraise
raise value.with_traceback(tb)
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\urllib3\connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\urllib3\connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\urllib3\connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\George\AppData\Local\Programs\Python\Python38-32\lib\http\client.py", line 1332, in getresponse
response.begin()
File "C:\Users\George\AppData\Local\Programs\Python\Python38-32\lib\http\client.py", line 303, in begin
version, status, reason = self._read_status()
File "C:\Users\George\AppData\Local\Programs\Python\Python38-32\lib\http\client.py", line 272, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\docker\api\client.py", line 214, in _retrieve_server_version
return self.version(api_version=False)["ApiVersion"]
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\docker\api\daemon.py", line 181, in version
return self._result(self._get(url), json=True)
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\docker\utils\decorators.py", line 46, in inner
return f(self, *args, **kwargs)
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\docker\api\client.py", line 237, in _get
return self.get(url, **self._set_request_timeout(kwargs))
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\requests\sessions.py", line 555, in get
return self.request('GET', url, **kwargs)
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\requests\sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\requests\sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\requests\adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:/Users/George/PycharmProjects/docker_project/docker_test.py", line 25, in <module>
client = docker.DockerClient(base_url ='ssh://george#192.168.56.103')
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\docker\client.py", line 45, in __init__
self.api = APIClient(*args, **kwargs)
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\docker\api\client.py", line 197, in __init__
self._version = self._retrieve_server_version()
File "C:\Users\George\PycharmProjects\docker_project\venv\lib\site-packages\docker\api\client.py", line 221, in _retrieve_server_version
raise DockerException(
docker.errors.DockerException: Error while fetching server API version: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
what could be the reason for that?
the problem was not with the connection itself. i tried to run the script localy on the remote server:
client = docker.from_env()
i had a permission denied exception.
the user must have sudo permission for the docker on the remote server to be able to use the Docker SDK. here more infos how to do that:
https://docs.docker.com/engine/install/linux-postinstall/
whenever it made the script works fine.
My scrapy application is outputting this long chain of exceptions and I am failing to see what the issue is and the last one has me especially confused.
Before I explain why here is the chain:
2020-11-04 17:38:58,394:ERROR:Error while obtaining start requests
Traceback (most recent call last):
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\urllib3\connectionpool.py", line 670, in urlopen
httplib_response = self._make_request(
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\urllib3\connectionpool.py", line 426, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\urllib3\connectionpool.py", line 421, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\http\client.py", line 1347, in getresponse
response.begin()
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\http\client.py", line 307, in begin
version, status, reason = self._read_status()
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\http\client.py", line 276, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\urllib3\connectionpool.py", line 726, in urlopen
retries = retries.increment(
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\urllib3\util\retry.py", line 403, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\urllib3\packages\six.py", line 734, in reraise
raise value.with_traceback(tb)
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\urllib3\connectionpool.py", line 670, in urlopen
httplib_response = self._make_request(
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\urllib3\connectionpool.py", line 426, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\urllib3\connectionpool.py", line 421, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\http\client.py", line 1347, in getresponse
response.begin()
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\http\client.py", line 307, in begin
version, status, reason = self._read_status()
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\http\client.py", line 276, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\shadow_useragent\core.py", line 35, in _update
r = requests.get(url=self.URL)
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\requests\api.py", line 76, in get
return request('get', url, params=params, **kwargs)
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\requests\api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\requests\sessions.py", line 530, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\requests\sessions.py", line 643, in send
r = adapter.send(request, **kwargs)
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\requests\adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\scrapy\core\engine.py", line 129, in _next_request
request = next(slot.start_requests)
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\scrapy_splash\middleware.py", line 167, in process_start_requests
for req in start_requests:
File "C:\Users\lguarro\Documents\Work\SearchEngine_Pure\SearchEngine_Pure\spiders\SearchEngine.py", line 36, in start_requests
user_agent = self.ua.random_nomobile
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\shadow_useragent\core.py", line 120, in random_nomobile
return self.pickrandom(exclude_mobile=True)
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\shadow_useragent\core.py", line 83, in pickrandom
self.update()
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\shadow_useragent\core.py", line 59, in update
self._update()
File "C:\Users\lguarro\Anaconda3\envs\virtual_workspace\lib\site-packages\shadow_useragent\core.py", line 38, in _update
self.logger.error(r.content.decode('utf-8'))
UnboundLocalError: local variable 'r' referenced before assignment
Now the last exception is complaining about some
UnboundLocalError: local variable 'r' referenced before assignment
The only code which is mine in that trace is the SearchEngine.py file which doesn't even have a variable 'r' thus leaving me very confused. Here is the implementation of start_requests from which the error occurs:
def start_requests(self):
user_agent = self.ua.random_nomobile # Exception raised here
rec = self.mh.FindIdleOneWithNoURLs()
if rec:
self.logger.info("Starting url scrape for company, %s using user agent: %s", rec["Company"], user_agent)
script = self.template.substitute(useragent=user_agent, searchquery=rec["Company"])
yield SplashRequest(url=self.url, callback=self.parse, endpoint="execute",
args={
'lua_source': script
},
meta={'RecID': rec["_id"], 'Company': rec["Company"]},
errback = self.logerror
)
It is complaining about the first line in that function for which I see no problem.
In case it is relevant, I will also add that my script seemed to be running fine just yesterday but today I had to reset my Docker configuration (that the splash container is running in) and since then I haven't been able to run my script smoothly.
I found out what was causing the issue! In fact there was no error on part, instead it is a bug inside the shadow-useragent library.
The library periodically makes an API request to fetch a list of the most used user agents and the server corresponding to this API is down and the authors of shadow-useragent were not properly handling the exception.
Fortunately shadow-useragent does cache the list of user agents that it was most recently able to receive. So my solution was to edit the shadow-useragent code to bypass the update function entirely and to use the cached list (inside the data.pk file) beyond its scheduled update. If anyone else runs into this issue, this is a temporary solution you can use until that server gets up and running again.. hopefully soon!
Working with OTRS system with PyOTRS trying to create ticket.
I get this message in response after I successfully create a ticket in the OTRS. I searched on the Internet for some people had a similar problem but found no solution anywhere.
Traceback (most recent call last):
File "/home/essaulov/try/try/lib/python3.5/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/home/essaulov/try/try/lib/python3.5/site-packages/urllib3/connectionpool.py", line 386, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "/home/essaulov/try/try/lib/python3.5/site-packages/urllib3/connectionpool.py", line 382, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.5/http/client.py", line 1197, in getresponse
response.begin()
File "/usr/lib/python3.5/http/client.py", line 297, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.5/http/client.py", line 266, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/essaulov/try/try/lib/python3.5/site-packages/requests/adapters.py", line 440, in send
timeout=timeout
File "/home/essaulov/try/try/lib/python3.5/site-packages/urllib3/connectionpool.py", line 649, in urlopen
_stacktrace=sys.exc_info()[2])
File "/home/essaulov/try/try/lib/python3.5/site-packages/urllib3/util/retry.py", line 357, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/essaulov/try/try/lib/python3.5/site-packages/urllib3/packages/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/home/essaulov/try/try/lib/python3.5/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/home/essaulov/try/try/lib/python3.5/site-packages/urllib3/connectionpool.py", line 386, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "/home/essaulov/try/try/lib/python3.5/site-packages/urllib3/connectionpool.py", line 382, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.5/http/client.py", line 1197, in getresponse
response.begin()
File "/usr/lib/python3.5/http/client.py", line 297, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.5/http/client.py", line 266, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/essaulov/try/try/lib/python3.5/site-packages/pyotrs/lib.py", line 1882, in _send_request
verify=self.https_verify)
File "/home/essaulov/try/try/lib/python3.5/site-packages/requests/api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "/home/essaulov/try/try/lib/python3.5/site-packages/requests/sessions.py", line 502, in request
resp = self.send(prep, **send_kwargs)
File "/home/essaulov/try/try/lib/python3.5/site-packages/requests/sessions.py", line 612, in send
r = adapter.send(request, **kwargs)
File "/home/essaulov/try/try/lib/python3.5/site-packages/requests/adapters.py", line 490, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "connetc.py", line 17, in <module>
client.ticket_create(new_ticket, first_article)
File "/home/essaulov/try/try/lib/python3.5/site-packages/pyotrs/lib.py", line 1125, in ticket_create
if not self._parse_and_validate_response(self._send_request(payload)):
File "/home/essaulov/try/try/lib/python3.5/site-packages/pyotrs/lib.py", line 1891, in _send_request
"Error with http communication: {0}".format(err))
pyotrs.lib.HTTPError: Failed to access OTRS. Check Hostname, Proxy, SSL Certificate!
Error with http communication: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))