I am trying to connect from Python to an Hadoop cluster that is running locally and I keep getting this error:
requests.exceptions.ConnectionError: ('Connection aborted.', BadStatusLine('\x00\x00\x00\x7f~\x08ÿÿÿÿ\x0f\x10\x02\x18\t")org.apache.hadoop.ipc.RPC$VersionMismatch*>Server IPC version 9 cannot communicate with client version
Here is my code:
`from hdfs import Config
class HDFSClient:
_instance = None
def __init__(self, env):
self._client = Config().get_client(env)
def write_data(self, data, hdfs_filename):
"""
Write data to an HDFS file.
"""
pass
def write_data_stream(self, iterator, hdfs_filename):
"""
Write data to an HDFS file.
Data is retrieved iteratively from #iterator.
"""
self._client.write(hdfs_filename, iterator)
#staticmethod
def get_instance(env="dev"):
if HDFSClient._instance:
return HDFSClient._instance
HDFSClient._instance = HDFSClient(env)
return HDFSClient._instance`
And my config file is very simple also:
[global]
default.alias = dev
[dev.alias]
url = http://localhost:9000
[prod.alias]
url = http://localhost:9000
Here is the full error:
Traceback (most recent call last):
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\connectionpool.py", line 436, in _make_request
httplib_response = conn.getresponse(buffering=True)
TypeError: getresponse() got an unexpected keyword argument 'buffering'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
# Permission is hereby granted, free of charge, to any person obtaining a copy
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\http\client.py", line 1347, in getresponse
response.begin()
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\http\client.py", line 307, in begin
version, status, reason = self._read_status()
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\http\client.py", line 289, in _read_status
raise BadStatusLine(line)
http.client.BadStatusLine:
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\util\retry.py", line 531, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\packages\six.py", line 734, in reraise
raise value.with_traceback(tb)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
# Permission is hereby granted, free of charge, to any person obtaining a copy
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\http\client.py", line 1347, in getresponse
response.begin()
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\http\client.py", line 307, in begin
version, status, reason = self._read_status()
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\http\client.py", line 289, in _read_status
raise BadStatusLine(line)
urllib3.exceptions.ProtocolError: ('Connection aborted.', BadStatusLine('\x00\x00\x00\x7f~\x08ÿÿÿÿ\x0f\x10\x02\x18\t")org.apache.hadoop.ipc.RPC$VersionMismatch*>Server IPC version 9 cannot communicate with client version 470\x0e:\x00#\x01Hõ*'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 2464, in __call__
return self.wsgi_app(environ, start_response)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 2450, in wsgi_app
response = self.handle_exception(e)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1867, in handle_exception
reraise(exc_type, exc_value, tb)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\_compat.py", line 39, in reraise
raise value
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1821, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\_compat.py", line 39, in reraise
raise value
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "C:\Users\Julien\Documents\Travail\PanoptesTechnologies\block-analysis-backend\api\src\with_cors.py", line 12, in decorated_function
return f(*args, **kwargs)
File "C:\Users\Julien\Documents\Travail\PanoptesTechnologies\block-analysis-backend\api\utils\log.py", line 37, in decorated
return f(*args, **kwargs)
File "C:\Users\Julien\Documents\Travail\PanoptesTechnologies\block-analysis-backend\api\app.py", line 154, in heuristic
#=======================================================================================================================
File "C:\Users\Julien\Documents\Travail\PanoptesTechnologies\block-analysis-backend\api\utils\hadoop.py", line 21, in write_data_stream
self._client.write(hdfs_filename, iterator)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\hdfs\client.py", line 459, in write
res = self._create(
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\hdfs\client.py", line 125, in api_handler
raise err
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\hdfs\client.py", line 102, in api_handler
res = client._request(
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\hdfs\client.py", line 209, in _request
return self._session.request(
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\requests\sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\requests\sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\requests\adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', BadStatusLine('\x00\x00\x00\x7f~\x08ÿÿÿÿ\x0f\x10\x02\x18\t")org.apache.hadoop.ipc.RPC$VersionMismatch*>Server IPC version 9 cannot communicate with client version 470\x0e:\x00#\x01Hõ*'))
Python version: 3.8
hdfscli version: 2.6.0
Hadoop version: 3.2.1
I already checked packages version needed by hdsfcli.
Is there something I can try to solve this?
Thank you for your help.
Best regads,
Welp, I had the same struggle.
For context:
This package uses the webhdfs API. On other versions, it was running on the 50070 port, but for Hadoop 3.2.1 and 3.3.1 (my version), it is running on the 9870 port.
Just change:
[dev.alias]
url = http://localhost:9000
[prod.alias]
url = http://localhost:9000
To:
[dev.alias]
url = http://localhost:9870
[prod.alias]
url = http://localhost:9870
And you're golden.
Related
I'm trying to make automated crypto monitoring program with python and using ccxt library.
(python version : 3.11)
After running this program for 6 to 7 hours, the following error occurred and the program was terminated.
Traceback (most recent call last):
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\connectionpool.py", line 703, in urlopen
httplib_response = self._make_request(
^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\connectionpool.py", line 449, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\connectionpool.py", line 444, in _make_request
httplib_response = conn.getresponse()
^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\http\client.py", line 1374, in getresponse
response.begin()
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\http\client.py", line 318, in begin
version, status, reason = self._read_status()
^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\http\client.py", line 287, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\requests\adapters.py", line 489, in send
resp = conn.urlopen(
^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\connectionpool.py", line 787, in urlopen
retries = retries.increment(
^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\util\retry.py", line 550, in increment
raise six.reraise(type(error), error, _stacktrace)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\packages\six.py", line 769, in reraise
raise value.with_traceback(tb)
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\connectionpool.py", line 703, in urlopen
httplib_response = self._make_request(
^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\connectionpool.py", line 449, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\urllib3\connectionpool.py", line 444, in _make_request
httplib_response = conn.getresponse()
^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\http\client.py", line 1374, in getresponse
response.begin()
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\http\client.py", line 318, in begin
version, status, reason = self._read_status()
^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\http\client.py", line 287, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\ccxt\base\exchange.py", line 599, in fetch
response = self.session.request(
^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\requests\sessions.py", line 587, in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\requests\sessions.py", line 701, in send
r = adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\requests\adapters.py", line 547, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "c:\Users\uid\Python\PJT_Crypto\Binance_Crypto_realtime.py", line 312, in <module>
Tracking_Crypto_Start_Start('15m', f, f2)
File "c:\Users\uid\Python\PJT_Crypto\Binance_Crypto_realtime.py", line 125, in Tracking_Crypto_Start_Start
get_data = bollinger_trade_logic(exchange, interval)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "c:\Users\uid\Python\PJT_Crypto\Binance_Crypto_realtime.py", line 29, in bollinger_trade_logic
symbol_df = get_data_frame(obj_binance, interval)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "c:\Users\uid\Python\PJT_Crypto\Binance_Crypto_realtime.py", line 19, in get_data_frame
p = obj_binance.fetch_ohlcv(
^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\ccxt\binance.py", line 2709, in fetch_ohlcv
response = getattr(self, method)(self.extend(request, params))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\ccxt\base\exchange.py", line 505, in inner
return entry(_self, **inner_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\ccxt\binance.py", line 6200, in request
response = self.fetch2(path, api, method, params, headers, body, config, context)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\ccxt\base\exchange.py", line 2830, in fetch2
return self.fetch(request['url'], request['method'], request['headers'], request['body'])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\uid\AppData\Local\Programs\Python\Python311\Lib\site-packages\ccxt\base\exchange.py", line 652, in fetch
raise NetworkError(details) from e
ccxt.base.errors.NetworkError: binance GET https://fapi.binance.com/fapi/v1/klines?interval=15m&limit=20&symbol=XRPUSDT
I think the Binance server forcibly rejected the request because I made too many requests.
Main code like this:
def get_data_frame(obj_binance, interval):
p = obj_binance.fetch_ohlcv(
'XRP/USDT', interval, limit=20)
df = pd.DataFrame(p, columns=['date', 'open',
'high', 'low', 'close', 'vol'])
return df
def Start_Monitoring(Param1,Param2):
try:
get_data = get_data_frame(Param1,Param2)
.....
...
time.sleep(1)
except ccxt.RequestTimeout as e:
print(e)
exchange = binanceperps(param) # This line for re-connect & request again. But never catched e untill now
There were many similar articles on the Internet, but I couldn't find a case that fits my case.
Please give me some advice.
Thanks
BR
I tried to add catch/exception code in while loop. But It couldn't capture the error.
The error what I post was occurred in python library not ccxt library.
I am trying to troubleshoot a situation.
I am initiating a Boto3 client like this:
s3_client = boto3.client('s3')
Then I am iterating over a number of files and uploading them using:
s3_client.upload_file()
My problem is that every now and then I see it pausing for 60 seconds and then continuing normally.
I've tried reducing the timeout value and the error I seem to be getting is:
botocore.exceptions.ConnectTimeoutError
or
urllib3.exceptions.ConnectTimeoutError
If I leave it at 60, eventually it succeeds and I don't get any errors.
My question is, this error means that when executing upload_file, it tried to send a put request to the server and the server didn't respond to that request?
When initiating the client, is any connection established that could be lost or that's simply only storing the credentials and it's irrelevant to this issue?
Many thanks.
Update: Adding more detailed logs:
CRITICAL Connect timeout
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 169, in _new_conn
conn = connection.create_connection(
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 96, in create_connection
raise err
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 86, in create_connection
sock.connect(sa)
TimeoutError: timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/user/.local/lib/python3.10/site-packages/botocore/httpsession.py", line 439, in send
urllib_response = conn.urlopen(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "/usr/lib/python3/dist-packages/urllib3/util/retry.py", line 507, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/lib/python3/dist-packages/six.py", line 719, in reraise
raise value
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 382, in _make_request
self._validate_conn(conn)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 1012, in _validate_conn
conn.connect()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 353, in connect
conn = self._new_conn()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 174, in _new_conn
raise ConnectTimeoutError(
urllib3.exceptions.ConnectTimeoutError
During handling of the above exception, another exception occurred:
File "/home/user/.local/lib/python3.10/site-packages/boto3/s3/inject.py", line 143, in upload_file
return transfer.upload_file(
File "/home/user/.local/lib/python3.10/site-packages/boto3/s3/transfer.py", line 288, in upload_file
future.result()
File "/home/user/.local/lib/python3.10/site-packages/s3transfer/futures.py", line 103, in result
return self._coordinator.result()
File "/home/user/.local/lib/python3.10/site-packages/s3transfer/futures.py", line 266, in result
raise self._exception
File "/home/user/.local/lib/python3.10/site-packages/s3transfer/tasks.py", line 139, in __call__
return self._execute_main(kwargs)
File "/home/user/.local/lib/python3.10/site-packages/s3transfer/tasks.py", line 162, in _execute_main
return_value = self._main(**kwargs)
File "/home/user/.local/lib/python3.10/site-packages/s3transfer/upload.py", line 758, in _main
client.put_object(Bucket=bucket, Key=key, Body=body, **extra_args)
File "/home/user/.local/lib/python3.10/site-packages/botocore/client.py", line 415, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/home/user/.local/lib/python3.10/site-packages/botocore/client.py", line 731, in _make_api_call
http, parsed_response = self._make_request(
File "/home/user/.local/lib/python3.10/site-packages/botocore/client.py", line 751, in _make_request
return self._endpoint.make_request(operation_model, request_dict)
File "/home/user/.local/lib/python3.10/site-packages/botocore/endpoint.py", line 107, in make_request
return self._send_request(request_dict, operation_model)
File "/home/user/.local/lib/python3.10/site-packages/botocore/endpoint.py", line 183, in _send_request
while self._needs_retry(attempts, operation_model, request_dict,
File "/home/user/.local/lib/python3.10/site-packages/botocore/endpoint.py", line 305, in _needs_retry
responses = self._event_emitter.emit(
File "/home/user/.local/lib/python3.10/site-packages/botocore/hooks.py", line 358, in emit
return self._emitter.emit(aliased_event_name, **kwargs)
File "/home/user/.local/lib/python3.10/site-packages/botocore/hooks.py", line 229, in emit
return self._emit(event_name, kwargs)
File "/home/user/.local/lib/python3.10/site-packages/botocore/hooks.py", line 212, in _emit
response = handler(**kwargs)
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 194, in __call__
if self._checker(**checker_kwargs):
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 267, in __call__
should_retry = self._should_retry(attempt_number, response,
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 294, in _should_retry
return self._checker(attempt_number, response, caught_exception)
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 333, in __call__
checker_response = checker(attempt_number, response,
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 233, in __call__
return self._check_caught_exception(
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 376, in _check_caught_exception
raise caught_exception
File "/home/user/.local/lib/python3.10/site-packages/botocore/endpoint.py", line 249, in _do_get_response
http_response = self._send(request)
File "/home/user/.local/lib/python3.10/site-packages/botocore/endpoint.py", line 321, in _send
return self.http_session.send(request)
File "/home/user/.local/lib/python3.10/site-packages/botocore/httpsession.py", line 472, in send
raise ConnectTimeoutError(endpoint_url=request.url, error=e)
The default boto3 retry mode (Legacy) doesn't properly handle a number of errors/exceptions, including RequestThrottled, PriorRequestNotComplete, ConnectionError and BandwidthLimitExceeded. It can manifest as very intermittent problems with long delays and repeated connection failures. You'll likely be better off using the Standard retry mode instead, which also has an exponential backoff.
import boto3
from botocore.client import Config as BotoConfig
TIMEOUT = 3
config = BotoConfig(connect_timeout=TIMEOUT, retries={"mode": "standard"})
client = boto3.client("s3", config=config)
I was using RemBg Software provided in this Repository. Whenever I used it. It gives me the following error-
Failed to import ahead-of-time-compiled modules.
This is expected on first import.
Compiling modules and trying again.
This might take a minute.
Traceback (most recent call last):
File "c:\python39\lib\site-packages\pymatting_aot\cc.py", line 36, in <module>
import pymatting_aot.aot
ModuleNotFoundError: No module named 'pymatting_aot.aot'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "c:\python39\lib\runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "c:\python39\lib\runpy.py", line 87, in _run_code
exec(code, run_globals)
File "C:\Python39\Scripts\rembg-server.exe\__main__.py", line 4, in <module>
File "c:\python39\lib\site-packages\rembg\cmd\server.py", line 11, in <module>
from ..bg import remove
File "c:\python39\lib\site-packages\rembg\bg.py", line 6, in <module>
from pymatting.alpha.estimate_alpha_cf import estimate_alpha_cf
File "c:\python39\lib\site-packages\pymatting\__init__.py", line 2, in <module>
import pymatting_aot.cc
File "c:\python39\lib\site-packages\pymatting_aot\cc.py", line 54, in <module>
compile_modules()
File "c:\python39\lib\site-packages\pymatting_aot\cc.py", line 8, in compile_modules
cc = CC("aot")
File "c:\python39\lib\site-packages\numba\pycc\cc.py", line 65, in __init__
self._toolchain = Toolchain()
File "c:\python39\lib\site-packages\numba\pycc\platform.py", line 78, in __init__
self._raise_external_compiler_error()
File "c:\python39\lib\site-packages\numba\pycc\platform.py", line 121, in _raise_external_compiler_error
raise RuntimeError(msg)
RuntimeError: Attempted to compile AOT function without the compiler used by `numpy.distutils` present. Cannot find suitable msvc.
I also tried to start their server by using the rembg-server command but it still not works.
Can anybody help me?
Update - Sorry for my behavior. I have installed C++ Build tools. But now when I run their "rembg-server" command it takes too much time to run almost 15 to 25 mins. and After the time it shows this error -
ERROR:rembg.cmd.server:Exception on / [GET]
Traceback (most recent call last):
File "c:\python39\lib\site-packages\flask\app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "c:\python39\lib\site-packages\flask\app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "c:\python39\lib\site-packages\flask\app.py", line 1821, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "c:\python39\lib\site-packages\flask\_compat.py", line 39, in reraise
raise value
File "c:\python39\lib\site-packages\flask\app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "c:\python39\lib\site-packages\flask\app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "c:\python39\lib\site-packages\rembg\cmd\server.py", line 31, in index
file_content = urlopen(unquote_plus(url)).read()
File "c:\python39\lib\urllib\request.py", line 214, in urlopen
return opener.open(url, data, timeout)
File "c:\python39\lib\urllib\request.py", line 517, in open
response = self._open(req, data)
File "c:\python39\lib\urllib\request.py", line 534, in _open
result = self._call_chain(self.handle_open, protocol, protocol +
File "c:\python39\lib\urllib\request.py", line 494, in _call_chain
result = func(*args)
File "c:\python39\lib\urllib\request.py", line 1375, in http_open
return self.do_open(http.client.HTTPConnection, req)
File "c:\python39\lib\urllib\request.py", line 1346, in do_open
h.request(req.get_method(), req.selector, req.data, headers,
File "c:\python39\lib\http\client.py", line 1253, in request
self._send_request(method, url, body, headers, encode_chunked)
File "c:\python39\lib\http\client.py", line 1264, in _send_request
self.putrequest(method, url, **skips)
File "c:\python39\lib\http\client.py", line 1098, in putrequest
self._validate_path(url)
File "c:\python39\lib\http\client.py", line 1198, in _validate_path
raise InvalidURL(f"URL can't contain control characters. {url!r} "
http.client.InvalidURL: URL can't contain control characters. '///C:/Users/nutan/Downloads/app anime icon [Music].jpeg' (found at least ' ')
ERROR:rembg.cmd.server:Exception on / [GET]
Traceback (most recent call last):
File "c:\python39\lib\urllib\request.py", line 1346, in do_open
h.request(req.get_method(), req.selector, req.data, headers,
File "c:\python39\lib\http\client.py", line 1253, in request
self._send_request(method, url, body, headers, encode_chunked)
File "c:\python39\lib\http\client.py", line 1299, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "c:\python39\lib\http\client.py", line 1248, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "c:\python39\lib\http\client.py", line 1008, in _send_output
self.send(msg)
File "c:\python39\lib\http\client.py", line 948, in send
self.connect()
File "c:\python39\lib\http\client.py", line 919, in connect
self.sock = self._create_connection(
File "c:\python39\lib\socket.py", line 822, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
File "c:\python39\lib\socket.py", line 953, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno 11001] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "c:\python39\lib\site-packages\flask\app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "c:\python39\lib\site-packages\flask\app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "c:\python39\lib\site-packages\flask\app.py", line 1821, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "c:\python39\lib\site-packages\flask\_compat.py", line 39, in reraise
raise value
File "c:\python39\lib\site-packages\flask\app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "c:\python39\lib\site-packages\flask\app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "c:\python39\lib\site-packages\rembg\cmd\server.py", line 31, in index
file_content = urlopen(unquote_plus(url)).read()
File "c:\python39\lib\urllib\request.py", line 214, in urlopen
return opener.open(url, data, timeout)
File "c:\python39\lib\urllib\request.py", line 517, in open
response = self._open(req, data)
File "c:\python39\lib\urllib\request.py", line 534, in _open
result = self._call_chain(self.handle_open, protocol, protocol +
File "c:\python39\lib\urllib\request.py", line 494, in _call_chain
result = func(*args)
File "c:\python39\lib\urllib\request.py", line 1375, in http_open
return self.do_open(http.client.HTTPConnection, req)
File "c:\python39\lib\urllib\request.py", line 1349, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 11001] getaddrinfo failed>
ERROR:rembg.cmd.server:Exception on / [GET]
Traceback (most recent call last):
File "c:\python39\lib\site-packages\flask\app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "c:\python39\lib\site-packages\flask\app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "c:\python39\lib\site-packages\flask\app.py", line 1821, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "c:\python39\lib\site-packages\flask\_compat.py", line 39, in reraise
raise value
File "c:\python39\lib\site-packages\flask\app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "c:\python39\lib\site-packages\flask\app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "c:\python39\lib\site-packages\rembg\cmd\server.py", line 31, in index
file_content = urlopen(unquote_plus(url)).read()
File "c:\python39\lib\urllib\request.py", line 214, in urlopen
return opener.open(url, data, timeout)
File "c:\python39\lib\urllib\request.py", line 514, in open
req = meth(req)
File "c:\python39\lib\urllib\request.py", line 1270, in do_request_
raise URLError('no host given')
urllib.error.URLError: <urlopen error no host given>
Edit - How can I use it using code. I wrote this code. Which is provided in their GitHub page. See the code and I am pasting the error below the code-
from rembg.bg import remove
import numpy as np
import io
from PIL import Image
input_path = './Images/IMG3.jpeg'
output_path = './Images/out3.jpeg'
f = np.fromfile(input_path)
result = remove(f)
img = Image.open(io.BytesIO(result)).convert("RGBA")
img.save(output_path)
Error message -
Traceback (most recent call last):
File "C:\Users\nutan\OneDrive\Desktop\Desktop\Locker\Image and Video Background Removal API and program\program\main.py", line 10, in <module>
result = remove(f)
File "C:\Python39\lib\site-packages\rembg\bg.py", line 91, in remove
img = Image.open(io.BytesIO(data)).convert("RGB")
File "C:\Python39\lib\site-packages\PIL\Image.py", line 904, in convert
self.load()
File "C:\Python39\lib\site-packages\PIL\ImageFile.py", line 249, in load
raise OSError(
OSError: image file is truncated (3 bytes not processed)
please fix these error above.
If somebody knows how to fix this please help. Thanks in Advance.
I got the same problem when install rembg 1.0.27 latest version. Later on, I have installed the previous version of rembg. and issues are resolved. you can try the older version of rembg. I have installed the 1.0.10 version and its working for me.
pip install rembg==1.0.10
Use this after imports packages
ImageFile.LOAD_TRUNCATED_IMAGES = True
Your problem will solved.
I want to download some files to my server's storage from URL. For example, you have url and want to save it. I've found only examples for downloading from the user's computer and it doesn't work in my case.
I used code below to download files in my local programs
with urllib.request.urlopen(url) as response, open(file_name, 'wb') as out_file:
shutil.copyfileobj(response, out_file)
but when running on server it produces this error:
Traceback (most recent call last):
File "/usr/lib/python3.8/site-packages/flask/app.py", line 2446, in wsgi_app
response = self.full_dispatch_request()
File "/usr/lib/python3.8/site-packages/flask/app.py", line 1951, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/lib/python3.8/site-packages/flask/app.py", line 1820, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/usr/lib/python3.8/site-packages/flask/_compat.py", line 39, in reraise
raise value
File "/usr/lib/python3.8/site-packages/flask/app.py", line 1949, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/lib/python3.8/site-packages/flask/app.py", line 1935, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/home/DenShlk/mysite/flask_app.py", line 26, in hello_world
save_pic_by_id(id)
File "/home/DenShlk/mysite/flask_app.py", line 20, in save_pic_by_id
with urllib.request.urlopen(pic_url) as response, open(file_name, 'wb') as out_file:
File "/usr/lib/python3.8/urllib/request.py", line 222, in urlopen
return opener.open(url, data, timeout)
File "/usr/lib/python3.8/urllib/request.py", line 525, in open
response = self._open(req, data)
File "/usr/lib/python3.8/urllib/request.py", line 542, in _open
result = self._call_chain(self.handle_open, protocol, protocol +
File "/usr/lib/python3.8/urllib/request.py", line 502, in _call_chain
result = func(*args)
File "/usr/lib/python3.8/urllib/request.py", line 1360, in https_open
return self.do_open(http.client.HTTPSConnection, req,
File "/usr/lib/python3.8/urllib/request.py", line 1320, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error Tunnel connection failed: 403 Forbidden>
Thanks for any help!
Finally, I've found the real problem. I used pythonanywhere.com and there is the white list of sites that can be accessed from free account.
I'm currently working on a project to do a timelapse with a raspberry pi3 (which works well, capturing the photos ain't a problem).
However, I used another script to upload those photos to a dropbox account and it work all well, until I checked it out today, seeing a timeout error.
Does anyone know how this could solved?
This is the error:
Traceback (most recent call last):
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/contrib/pyopenssl.py", line 453, in wrap_socket
cnx.do_handshake()
File "/usr/lib/python3/dist-packages/OpenSSL/SSL.py", line 1426, in do_handshake
self._raise_ssl_error(self._ssl, result)
File "/usr/lib/python3/dist-packages/OpenSSL/SSL.py", line 1149, in _raise_ssl_error
raise WantReadError()
OpenSSL.SSL.WantReadError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 343, in _make_request
self._validate_conn(conn)
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 839, in _validate_conn
conn.connect()
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/connection.py", line 344, in connect
ssl_context=context)
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/util/ssl_.py", line 344, in ssl_wrap_socket
return context.wrap_socket(sock, server_hostname=server_hostname)
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/contrib/pyopenssl.py", line 456, in wrap_socket
raise timeout('select timed out')
socket.timeout: select timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/pi/.local/lib/python3.5/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/util/retry.py", line 367, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/packages/six.py", line 686, in reraise
raise value
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 346, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 306, in _raise_timeout
raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='content.dropboxapi.com', port=443): Read timed out. (read timeout=30)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/pi/Desktop/time/picture/drop.py", line 16, in <module>
response = db.files_upload(f.read(), dname)
File "/home/pi/.local/lib/python3.5/site-packages/dropbox/base.py", line 2293, in files_upload
f,
File "/home/pi/.local/lib/python3.5/site-packages/dropbox/dropbox.py", line 274, in request
timeout=timeout)
File "/home/pi/.local/lib/python3.5/site-packages/dropbox/dropbox.py", line 365, in request_json_string_with_retry
timeout=timeout)
File "/home/pi/.local/lib/python3.5/site-packages/dropbox/dropbox.py", line 449, in request_json_string
timeout=timeout,
File "/home/pi/.local/lib/python3.5/site-packages/requests/sessions.py", line 581, in post
return self.request('POST', url, data=data, json=json, **kwargs)
File "/home/pi/.local/lib/python3.5/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/home/pi/.local/lib/python3.5/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/home/pi/.local/lib/python3.5/site-packages/requests/adapters.py", line 529, in send
raise ReadTimeout(e, request=request)
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='content.dropboxapi.com', port=443): Read timed out. (read timeout=30)
>>>
This is how the code to upload on dropbox looks:
import dropbox
from time import sleep
db = dropbox.Dropbox('apikeyplaceholder')
for i in range(4500): ##4500
name = 'image{0:04d}.jpg'
fname = '/home/pi/Desktop/time/picture/'+name.format(i)
dname = '/'+name.format(i)
f = open(fname, 'rb')
response = db.files_upload(f.read(), dname)
print ('uploaded:', response)
sleep(585)
f.close()
NEW ERROR:
Traceback (most recent call last):
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/contrib/pyopenssl.py", line 317, in _send_until_done
return self.connection.send(data)
File "/usr/lib/python3/dist-packages/OpenSSL/SSL.py", line 1256, in send
self._raise_ssl_error(self._ssl, result)
File "/usr/lib/python3/dist-packages/OpenSSL/SSL.py", line 1151, in _raise_ssl_error
raise WantWriteError()
OpenSSL.SSL.WantWriteError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib/python3.5/http/client.py", line 1107, in request
self._send_request(method, url, body, headers)
File "/usr/lib/python3.5/http/client.py", line 1152, in _send_request
self.endheaders(body)
File "/usr/lib/python3.5/http/client.py", line 1103, in endheaders
self._send_output(message_body)
File "/usr/lib/python3.5/http/client.py", line 936, in _send_output
self.send(message_body)
File "/usr/lib/python3.5/http/client.py", line 908, in send
self.sock.sendall(data)
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/contrib/pyopenssl.py", line 328, in sendall
sent = self._send_until_done(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE])
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/contrib/pyopenssl.py", line 320, in _send_until_done
raise timeout()
socket.timeout
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/pi/.local/lib/python3.5/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/util/retry.py", line 367, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/packages/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib/python3.5/http/client.py", line 1107, in request
self._send_request(method, url, body, headers)
File "/usr/lib/python3.5/http/client.py", line 1152, in _send_request
self.endheaders(body)
File "/usr/lib/python3.5/http/client.py", line 1103, in endheaders
self._send_output(message_body)
File "/usr/lib/python3.5/http/client.py", line 936, in _send_output
self.send(message_body)
File "/usr/lib/python3.5/http/client.py", line 908, in send
self.sock.sendall(data)
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/contrib/pyopenssl.py", line 328, in sendall
sent = self._send_until_done(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE])
File "/home/pi/.local/lib/python3.5/site-packages/urllib3/contrib/pyopenssl.py", line 320, in _send_until_done
raise timeout()
urllib3.exceptions.ProtocolError: ('Connection aborted.', timeout())
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/pi/Desktop/time/bild/drop.py", line 20, in <module>
response = upload(f, dname)
File "/usr/local/lib/python3.5/dist-packages/backoff/_sync.py", line 94, in retry
ret = target(*args, **kwargs)
File "/home/pi/Desktop/time/bild/drop.py", line 11, in upload
return db.files_upload(f.read(), dname)
File "/home/pi/.local/lib/python3.5/site-packages/dropbox/base.py", line 2293, in files_upload
f,
File "/home/pi/.local/lib/python3.5/site-packages/dropbox/dropbox.py", line 274, in request
timeout=timeout)
File "/home/pi/.local/lib/python3.5/site-packages/dropbox/dropbox.py", line 365, in request_json_string_with_retry
timeout=timeout)
File "/home/pi/.local/lib/python3.5/site-packages/dropbox/dropbox.py", line 449, in request_json_string
timeout=timeout,
File "/home/pi/.local/lib/python3.5/site-packages/requests/sessions.py", line 581, in post
return self.request('POST', url, data=data, json=json, **kwargs)
File "/home/pi/.local/lib/python3.5/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/home/pi/.local/lib/python3.5/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/home/pi/.local/lib/python3.5/site-packages/requests/adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', timeout())
Before you implement code to handle the timeout you first need to think about how you want to handle it. A common strategy is the following:
Try re-uploading a certain number of times, potentially coupled with exponential backoff (e.g. using this library)
If that fails either skip the current image or abort the process completely.
Depending on your setup, you might want to put the uploading code into a separate thread so that delays during uploads don't interfere with capturing the photos.
A simple example using the backoff library I've linked to above (untested):
from time import sleep
import backoff
import dropbox
import requests
db = dropbox.Dropbox('apikeyplaceholder')
#backoff.on_exception(backoff.expo, requests.exceptions.ReadTimeout)
def upload(f, name):
return db.files_upload(f.read(), dname)
NAME_PATTERN = '/image{0:04d}.jpg'
for i in range(4500):
name = NAME_PATTERN.format(i)
fname = '/home/pi/Desktop/time/picture' + name
with open(fname, 'rb') as f:
response = upload(f, name)
print ('uploaded:', response)
sleep(585)