Keep Running a tweepy stream for a long time - python

I'm running a python script that is supposed to listen tweets with a specific '#' (and then execute some code).
My code is almost running 100% correctly but I still have an issue: my program stops running at random time for "no reason".
Here is the error:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/http/client.py", line 551, in _get_chunk_left
chunk_left = self._read_next_chunk_size()
File "/usr/local/lib/python3.8/http/client.py", line 518, in _read_next_chunk_size
return int(line, 16)
ValueError: invalid literal for int() with base 16: b''
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/http/client.py", line 583, in _readinto_chunked
chunk_left = self._get_chunk_left()
File "/usr/local/lib/python3.8/http/client.py", line 553, in _get_chunk_left
raise IncompleteRead(b'')
http.client.IncompleteRead: IncompleteRead(0 bytes read)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/urllib3/response.py", line 436, in _error_catcher
yield
File "/usr/local/lib/python3.8/site-packages/urllib3/response.py", line 518, in read
data = self._fp.read(amt) if not fp_closed else b""
File "/usr/local/lib/python3.8/http/client.py", line 454, in read
n = self.readinto(b)
File "/usr/local/lib/python3.8/http/client.py", line 488, in readinto
return self._readinto_chunked(b)
File "/usr/local/lib/python3.8/http/client.py", line 599, in _readinto_chunked
raise IncompleteRead(bytes(b[0:total_bytes]))
http.client.IncompleteRead: IncompleteRead(292 bytes read)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "main.py", line 133, in <module>
myStream.filter(track=['#DownDropship'])
File "/usr/local/lib/python3.8/site-packages/tweepy/streaming.py", line 474, in filter
self._start(is_async)
File "/usr/local/lib/python3.8/site-packages/tweepy/streaming.py", line 389, in _start
self._run()
File "/usr/local/lib/python3.8/site-packages/tweepy/streaming.py", line 320, in _run
six.reraise(*exc_info)
File "/usr/local/lib/python3.8/site-packages/six.py", line 703, in reraise
raise value
File "/usr/local/lib/python3.8/site-packages/tweepy/streaming.py", line 289, in _run
self._read_loop(resp)
File "/usr/local/lib/python3.8/site-packages/tweepy/streaming.py", line 339, in _read_loop
line = buf.read_line()
File "/usr/local/lib/python3.8/site-packages/tweepy/streaming.py", line 200, in read_line
self._buffer += self._stream.read(self._chunk_size)
File "/usr/local/lib/python3.8/site-packages/urllib3/response.py", line 540, in read
raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
File "/usr/local/lib/python3.8/contextlib.py", line 131, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/local/lib/python3.8/site-packages/urllib3/response.py", line 454, in _error_catcher
raise ProtocolError("Connection broken: %r" % e, e)
urllib3.exceptions.ProtocolError: ('Connection broken: IncompleteRead(292 bytes read)', IncompleteRead(292 bytes read))
Here is a piece of my code:
from tweepy import StreamListener
from retrying import retry
import tweepy
#retry
class MyStreamListener(StreamListener):
#retry
def on_status(self, status):
do something ...
except Exception as e:
pass
myStreamListener = MyStreamListener()
myStream = tweepy.Stream(auth=twitter_handler.api.auth, listener=myStreamListener)
myStream.filter(track=['#Something'])
I found these "#retry" here but apparently, it doesn't work properly.
Thanks for your help

Related

How to trap this nested httpx exception?

I'm trapping thus:
with httpx.Client(**sessions[scraperIndex]) as client:
try:
response = client.get(...)
except TimeoutError as e:
print('does not hit')
except Exception as e:
print(f'⛔️ Unexpected exception: {e}')
print_exc() # hits!
However I'm getting the below crashdump.
Pulling out key lines:
TimeoutError: The read operation timed out
During handling of the above exception, another exception occurred:
httpcore.ReadTimeout: The read operation timed out
The above exception was the direct cause of the following exception:
httpx.ReadTimeout: The read operation timed out
Why isn't my TimeoutError catching this?
And what's the correct catch? Can someone give a logic for deducing it?
CrashDump:
⛔️ Unexpected exception: The read operation timed out
Traceback (most recent call last):
File "/usr/local/lib/python3.10/dist-packages/httpcore/_exceptions.py", line 8, in map_exceptions
yield
File "/usr/local/lib/python3.10/dist-packages/httpcore/backends/sync.py", line 26, in read
return self._sock.recv(max_bytes)
File "/usr/lib/python3.10/ssl.py", line 1258, in recv
return self.read(buflen)
File "/usr/lib/python3.10/ssl.py", line 1131, in read
return self._sslobj.read(len)
TimeoutError: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.10/dist-packages/httpx/_transports/default.py", line 60, in map_httpcore_exceptions
yield
File "/usr/local/lib/python3.10/dist-packages/httpx/_transports/default.py", line 218, in handle_request
resp = self._pool.handle_request(req)
File "/usr/local/lib/python3.10/dist-packages/httpcore/_sync/connection_pool.py", line 253, in handle_request
raise exc
File "/usr/local/lib/python3.10/dist-packages/httpcore/_sync/connection_pool.py", line 237, in handle_request
response = connection.handle_request(request)
File "/usr/local/lib/python3.10/dist-packages/httpcore/_sync/connection.py", line 90, in handle_request
return self._connection.handle_request(request)
File "/usr/local/lib/python3.10/dist-packages/httpcore/_sync/http11.py", line 105, in handle_request
raise exc
File "/usr/local/lib/python3.10/dist-packages/httpcore/_sync/http11.py", line 84, in handle_request
) = self._receive_response_headers(**kwargs)
File "/usr/local/lib/python3.10/dist-packages/httpcore/_sync/http11.py", line 148, in _receive_response_headers
event = self._receive_event(timeout=timeout)
File "/usr/local/lib/python3.10/dist-packages/httpcore/_sync/http11.py", line 177, in _receive_event
data = self._network_stream.read(
File "/usr/local/lib/python3.10/dist-packages/httpcore/backends/sync.py", line 24, in read
with map_exceptions(exc_map):
File "/usr/lib/python3.10/contextlib.py", line 153, in __exit__
self.gen.throw(typ, value, traceback)
File "/usr/local/lib/python3.10/dist-packages/httpcore/_exceptions.py", line 12, in map_exceptions
raise to_exc(exc)
httpcore.ReadTimeout: The read operation timed out
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/root/scraper-pi/Scrape.py", line 148, in main
cursor, _nScraped = scrape(client, cursor)
File "/root/scraper-pi/Scrape.py", line 79, in scrape
response = client.get(
File "/usr/local/lib/python3.10/dist-packages/httpx/_client.py", line 1039, in get
return self.request(
File "/usr/local/lib/python3.10/dist-packages/httpx/_client.py", line 815, in request
return self.send(request, auth=auth, follow_redirects=follow_redirects)
File "/usr/local/lib/python3.10/dist-packages/httpx/_client.py", line 902, in send
response = self._send_handling_auth(
File "/usr/local/lib/python3.10/dist-packages/httpx/_client.py", line 930, in _send_handling_auth
response = self._send_handling_redirects(
File "/usr/local/lib/python3.10/dist-packages/httpx/_client.py", line 967, in _send_handling_redirects
response = self._send_single_request(request)
File "/usr/local/lib/python3.10/dist-packages/httpx/_client.py", line 1003, in _send_single_request
response = transport.handle_request(request)
File "/usr/local/lib/python3.10/dist-packages/httpx/_transports/default.py", line 217, in handle_request
with map_httpcore_exceptions():
File "/usr/lib/python3.10/contextlib.py", line 153, in __exit__
self.gen.throw(typ, value, traceback)
File "/usr/local/lib/python3.10/dist-packages/httpx/_transports/default.py", line 77, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.ReadTimeout: The read operation timed out

The base class for all httpx timeout errors is not the built-in TimeoutError (presumably because that would also make timeouts OSErrors, which doesn't sound correct), but httpx.TimeoutException.
import httpx
with httpx.Client() as client:
try:
response = client.get("http://httpbin.org/get", timeout=0.001)
except httpx.TimeoutException as e:
print('gottem')
prints gottem just fine.

Python SpeechRegonition

I write a little program which writes the audio to text. But it throws an error and I don't know how to fix it.
import speech_recognition as sr
file_name = "halloWelt.wav"
speech_engine = sr.Recognizer()
with sr.AudioFile(file_name) as file:
data = speech_engine.record()
text = speech_engine.recognize_google(data, language='de-DE')
print(text)
The error:
Traceback (most recent call last):
File "C:\Users\\AppData\Local\Programs\Python\Python310\lib\site-packages\speech_recognition\__init__.py", line 203, in __enter__
self.audio_reader = wave.open(self.filename_or_fileobject, "rb")
File "C:\Users\\AppData\Local\Programs\Python\Python310\lib\wave.py", line 509, in open
return Wave_read(f)
File "C:\Users\\AppData\Local\Programs\Python\Python310\lib\wave.py", line 163, in __init__
self.initfp(f)
File "C:\Users\\AppData\Local\Programs\Python\Python310\lib\wave.py", line 130, in initfp
raise Error('file does not start with RIFF id')
wave.Error: file does not start with RIFF id
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\\AppData\Local\Programs\Python\Python310\lib\site-packages\speech_recognition\__init__.py", line 208, in __enter__
self.audio_reader = aifc.open(self.filename_or_fileobject, "rb")
File "C:\Users\\AppData\Local\Programs\Python\Python310\lib\aifc.py", line 917, in open
return Aifc_read(f)
File "C:\Users\\AppData\Local\Programs\Python\Python310\lib\aifc.py", line 352, in __init__
self.initfp(file_object)
File "C:\Users\\AppData\Local\Programs\Python\Python310\lib\aifc.py", line 316, in initfp
raise Error('file does not start with FORM id')
aifc.Error: file does not start with FORM id
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\\AppData\Local\Programs\Python\Python310\lib\site-packages\speech_recognition\__init__.py", line 234, in __enter__
self.audio_reader = aifc.open(aiff_file, "rb")
File "C:\Users\\AppData\Local\Programs\Python\Python310\lib\aifc.py", line 917, in open
return Aifc_read(f)
File "C:\Users\\AppData\Local\Programs\Python\Python310\lib\aifc.py", line 358, in __init__
self.initfp(f)
File "C:\Users\\AppData\Local\Programs\Python\Python310\lib\aifc.py", line 314, in initfp
chunk = Chunk(file)
File "C:\Users\\AppData\Local\Programs\Python\Python310\lib\chunk.py", line 63, in __init__
raise EOFError
EOFError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "c:\Users\\OneDrive\Dokumente\Programming\Python\Lets code\speech.py", line 6, in <module>
with sr.AudioFile(file_name) as file:
File "C:\Users\\AppData\Local\Programs\Python\Python310\lib\site-packages\speech_recognition\__init__.py", line 236, in __enter__
raise ValueError("Audio file could not be read as PCM WAV, AIFF/AIFF-C, or Native FLAC; check if file is corrupted or in another format")
ValueError: Audio file could not be read as PCM WAV, AIFF/AIFF-C, or Native FLAC; check if file is corrupted or in another format

Error during execution: docker-compose run --rm api python3 manage.py migrate

I am trying to install saleor on Linux Mint according to the instructions
https://docs.saleor.io/docs/3.0/developer/installation
When executing the command
docker-compose run --rm api python3 manage.py migrate
I get an error. This is the stack trace. How do I resolve this issue?
$docker-compose run --rm api python3 manage.py migrate
Starting saleor-platform_db_1 ...
Starting saleor-platform_jaeger_1 ... done
Starting saleor-platform_redis_1 ... done
ERROR: for saleor-platform_db_1 a bytes-like object is required, not 'str'
ERROR: for db a bytes-like object is required, not 'str'
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/docker/api/client.py", line 261, in _raise_for_status
response.raise_for_status()
File "/usr/lib/python3/dist-packages/requests/models.py", line 940, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 500 Server Error: Internal Server Error for url: http+docker://localhost/v1.22/containers/c015b9d2a6e0ba06c8cc393147db2a4eb1a0fc72d1ae2805e177b409bb8212db/start
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/compose/service.py", line 625, in start_container
container.start()
File "/usr/lib/python3/dist-packages/compose/container.py", line 241, in start
return self.client.start(self.id, **options)
File "/usr/lib/python3/dist-packages/docker/utils/decorators.py", line 19, in wrapped
return f(self, resource_id, *args, **kwargs)
File "/usr/lib/python3/dist-packages/docker/api/container.py", line 1095, in start
self._raise_for_status(res)
File "/usr/lib/python3/dist-packages/docker/api/client.py", line 263, in _raise_for_status
raise create_api_error_from_http_exception(e)
File "/usr/lib/python3/dist-packages/docker/errors.py", line 31, in create_api_error_from_http_exception
raise cls(e, response=response, explanation=explanation)
docker.errors.APIError: 500 Server Error: Internal Server Error ("b'driver failed programming external connectivity on endpoint saleor-platform_db_1 (1b57cb27e18e4e18fad1fde3f6bebb573260974514be140c7e4e0d74d663b7b0): Error starting userland proxy: listen tcp4 0.0.0.0:5432: bind: address already in use'")
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/bin/docker-compose", line 11, in <module>
load_entry_point('docker-compose==1.25.0', 'console_scripts', 'docker-compose')()
File "/usr/lib/python3/dist-packages/compose/cli/main.py", line 72, in main
command()
File "/usr/lib/python3/dist-packages/compose/cli/main.py", line 128, in perform_command
handler(command, command_options)
File "/usr/lib/python3/dist-packages/compose/cli/main.py", line 896, in run
run_one_off_container(
File "/usr/lib/python3/dist-packages/compose/cli/main.py", line 1343, in run_one_off_container
project.up(
File "/usr/lib/python3/dist-packages/compose/project.py", line 565, in up
results, errors = parallel.parallel_execute(
File "/usr/lib/python3/dist-packages/compose/parallel.py", line 112, in parallel_execute
raise error_to_reraise
File "/usr/lib/python3/dist-packages/compose/parallel.py", line 210, in producer
result = func(obj)
File "/usr/lib/python3/dist-packages/compose/project.py", line 548, in do
return service.execute_convergence_plan(
File "/usr/lib/python3/dist-packages/compose/service.py", line 567, in execute_convergence_plan
return self._execute_convergence_start(
File "/usr/lib/python3/dist-packages/compose/service.py", line 506, in _execute_convergence_start
_, errors = parallel_execute(
File "/usr/lib/python3/dist-packages/compose/parallel.py", line 112, in parallel_execute
raise error_to_reraise
File "/usr/lib/python3/dist-packages/compose/parallel.py", line 210, in producer
result = func(obj)
File "/usr/lib/python3/dist-packages/compose/service.py", line 508, in <lambda>
lambda c: self.start_container_if_stopped(c, attach_logs=not detached, quiet=True),
File "/usr/lib/python3/dist-packages/compose/service.py", line 620, in start_container_if_stopped
return self.start_container(container)
File "/usr/lib/python3/dist-packages/compose/service.py", line 627, in start_container
if "driver failed programming external connectivity" in ex.explanation:
TypeError: a bytes-like object is required, not 'str'
Error in sys.excepthook:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/apport_python_hook.py", line 153, in apport_excepthook
with os.fdopen(os.open(pr_filename,
FileNotFoundError: [Errno 2] No such file or directory: '/var/crash/_usr_bin_docker-compose.1000.crash'
Original exception was:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/docker/api/client.py", line 261, in _raise_for_status
response.raise_for_status()
File "/usr/lib/python3/dist-packages/requests/models.py", line 940, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 500 Server Error: Internal Server Error for url: http+docker://localhost/v1.22/containers/c015b9d2a6e0ba06c8cc393147db2a4eb1a0fc72d1ae2805e177b409bb8212db/start
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/compose/service.py", line 625, in start_container
container.start()
File "/usr/lib/python3/dist-packages/compose/container.py", line 241, in start
return self.client.start(self.id, **options)
File "/usr/lib/python3/dist-packages/docker/utils/decorators.py", line 19, in wrapped
return f(self, resource_id, *args, **kwargs)
File "/usr/lib/python3/dist-packages/docker/api/container.py", line 1095, in start
self._raise_for_status(res)
File "/usr/lib/python3/dist-packages/docker/api/client.py", line 263, in _raise_for_status
raise create_api_error_from_http_exception(e)
File "/usr/lib/python3/dist-packages/docker/errors.py", line 31, in create_api_error_from_http_exception
raise cls(e, response=response, explanation=explanation)
docker.errors.APIError: 500 Server Error: Internal Server Error ("b'driver failed programming external connectivity on endpoint saleor-platform_db_1 (1b57cb27e18e4e18fad1fde3f6bebb573260974514be140c7e4e0d74d663b7b0): Error starting userland proxy: listen tcp4 0.0.0.0:5432: bind: address already in use'")
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/bin/docker-compose", line 11, in <module>
load_entry_point('docker-compose==1.25.0', 'console_scripts', 'docker-compose')()
File "/usr/lib/python3/dist-packages/compose/cli/main.py", line 72, in main
command()
File "/usr/lib/python3/dist-packages/compose/cli/main.py", line 128, in perform_command
handler(command, command_options)
File "/usr/lib/python3/dist-packages/compose/cli/main.py", line 896, in run
run_one_off_container(
File "/usr/lib/python3/dist-packages/compose/cli/main.py", line 1343, in run_one_off_container
project.up(
File "/usr/lib/python3/dist-packages/compose/project.py", line 565, in up
results, errors = parallel.parallel_execute(
File "/usr/lib/python3/dist-packages/compose/parallel.py", line 112, in parallel_execute
raise error_to_reraise
File "/usr/lib/python3/dist-packages/compose/parallel.py", line 210, in producer
result = func(obj)
File "/usr/lib/python3/dist-packages/compose/project.py", line 548, in do
return service.execute_convergence_plan(
File "/usr/lib/python3/dist-packages/compose/service.py", line 567, in execute_convergence_plan
return self._execute_convergence_start(
File "/usr/lib/python3/dist-packages/compose/service.py", line 506, in _execute_convergence_start
_, errors = parallel_execute(
File "/usr/lib/python3/dist-packages/compose/parallel.py", line 112, in parallel_execute
raise error_to_reraise
File "/usr/lib/python3/dist-packages/compose/parallel.py", line 210, in producer
result = func(obj)
File "/usr/lib/python3/dist-packages/compose/service.py", line 508, in <lambda>
lambda c: self.start_container_if_stopped(c, attach_logs=not detached, quiet=True),
File "/usr/lib/python3/dist-packages/compose/service.py", line 620, in start_container_if_stopped
return self.start_container(container)
File "/usr/lib/python3/dist-packages/compose/service.py", line 627, in start_container
if "driver failed programming external connectivity" in ex.explanation:
TypeError: a bytes-like object is required, not 'str'

Download the spacy model fr_core_news_lg

I use spacy to do some NLP tasks but every time I try to download the word embbeding model from spacy
fr_core_news_lg,
there is:
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred
This error stops the model download execution.
(base) C:\Users\marct>python -m spacy download fr_core_news_lg
Collecting fr_core_news_lg==2.3.0 from https://github.com/explosion/spacy-models/releases/download/fr_core_news_lg-2.3.0/fr_core_news_lg-2.3.0.tar.gz#egg=fr_core_news_lg==2.3.0
Downloading https://github.com/explosion/spacy-models/releases/download/fr_core_news_lg-2.3.0/fr_core_news_lg-2.3.0.tar.gz (572.0MB)
85% |███████████████████████████ | 487.3MB 27kB/s eta 0:52:09Exception:
Traceback (most recent call last):
File "C:\Users\marct\Anaconda3\lib\site-packages\pip\_vendor\urllib3\response.py", line 360, in _error_catcher
yield
File "C:\Users\marct\Anaconda3\lib\site-packages\pip\_vendor\urllib3\response.py", line 442, in read
data = self._fp.read(amt)
return self._sslobj.read(len, buffer)
File "C:\Users\marct\Anaconda3\lib\ssl.py", line 631, in read
v = self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\marct\Anaconda3\lib\site-packages\pip\_internal\cli\base_command.py", line 179, in main
status = self.run(options, args)
File "C:\Users\marct\Anaconda3\lib\site-packages\pip\_internal\download.py", line 946, in _download_http_url
_download_url(resp, link, content_file, hashes, progress_bar)
File "C:\Users\marct\Anaconda3\lib\site-packages\pip\_internal\download.py", line 641, in _download_url
consume(downloaded_chunks)
File "C:\Users\marct\Anaconda3\lib\site-packages\pip\_internal\utils\misc.py", line 914, in consume
deque(iterator, maxlen=0)
File "C:\Users\marct\Anaconda3\lib\site-packages\pip\_internal\download.py", line 607, in written_chunks
for chunk in chunks:
File "C:\Users\marct\Anaconda3\lib\site-packages\pip\_internal\utils\ui.py", line 159, in iter
for x in it:
File "C:\Users\marct\Anaconda3\lib\site-packages\pip\_internal\download.py", line 596, in resp_read
decode_content=False):
File "C:\Users\marct\Anaconda3\lib\site-packages\pip\_vendor\urllib3\response.py", line 494, in stream
data = self.read(amt=amt, decode_content=decode_content)
File "C:\Users\marct\Anaconda3\lib\site-packages\pip\_vendor\urllib3\response.py", line 459, in read
raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
File "C:\Users\marct\Anaconda3\lib\contextlib.py", line 99, in __exit__
self.gen.throw(type, value, traceback)
File "C:\Users\marct\Anaconda3\lib\site-packages\pip\_vendor\urllib3\response.py", line 365, in _error_catcher
raise ReadTimeoutError(self._pool, None, 'Read timed out.')
pip._vendor.urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='github-production-release-asset-2e65be.s3.amazonaws.com', port=443): Read timed out.
If there is someone who can help me with the problem.
Thank you in advance

Pycharm debugger throws Bad file descriptor error when using dask distributed

I am using the most lightweight/simple dask multiprocessing which is the non-cluster local Client:
from distributed import Client
client = Client()
Even so: the first instance of invoking dask.bag.compute() results in the following:
Connected to pydev debugger (build 191.7141.48)
Traceback (most recent call last):
File "/Applications/PyCharm.app/Contents/helpers/pydev/_pydevd_bundle/pydevd_comm.py", line 383, in _on_run
r = self.sock.recv(1024)
OSError: [Errno 9] Bad file descriptor
Traceback (most recent call last):
File "/Applications/PyCharm.app/Contents/helpers/pydev/_pydevd_bundle/pydevd_comm.py", line 383, in _on_run
r = self.sock.recv(1024)
OSError: [Errno 9] Bad file descriptor
Traceback (most recent call last):
File "/Applications/PyCharm.app/Contents/helpers/pydev/_pydevd_bundle/pydevd_comm.py", line 383, in _on_run
r = self.sock.recv(1024)
OSError: [Errno 9] Bad file descriptor
The result is that you can more or less flip a coin on whether the program will proceed or error out with a communication exception. Here is what happens when the flip comes up "tails":
Connected to pydev debugger (build 191.7141.48)
Traceback (most recent call last):
File "/Applications/PyCharm.app/Contents/helpers/pydev/_pydevd_bundle/pydevd_comm.py", line 383, in _on_run
r = self.sock.recv(1024)
OSError: [Errno 9] Bad file descriptor
Process ForkServerProcess-3:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/distributed/core.py", line 178, in __init__
from .counter import Digest
ImportError: cannot import name 'Digest' from 'distributed.counter' (/usr/local/lib/python3.7/site-packages/distributed/counter.py)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/Cellar/python/3.7.3/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
self.run()
File "/usr/local/Cellar/python/3.7.3/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.7/site-packages/distributed/process.py", line 181, in _run
target(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/distributed/nanny.py", line 587, in _run
worker = Worker(*worker_args, **worker_kwargs)
File "/usr/local/lib/python3.7/site-packages/distributed/worker.py", line 552, in __init__
**kwargs
File "/usr/local/lib/python3.7/site-packages/distributed/node.py", line 76, in __init__
io_loop=self.io_loop,
File "/usr/local/lib/python3.7/site-packages/distributed/core.py", line 180, in __init__
self.digests = defaultdict(partial(Digest, loop=self.io_loop))
File "/usr/local/Cellar/python/3.7.3/Frameworks/Python.framework/Versions/3.7/lib/python3.7/contextlib.py", line 130, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/local/lib/python3.7/site-packages/distributed/utils.py", line 179, in ignoring
yield
SystemError: error return without exception set
distributed.nanny - WARNING - Worker process 20417 exited with status 1
Traceback (most recent call last):
File "_pydevd_frame_eval/pydevd_frame_evaluator_darwin_37_64.pyx", line 95, in _pydevd_frame_eval.pydevd_frame_evaluator_darwin_37_64.get_bytecode_while_frame_eval
KeyError: '/usr/local/lib/python3.7/site-packages/distributed/bokeh/__init__.py'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Applications/PyCharm.app/Contents/helpers/pydev/pydevd.py", line 1758, in <module>
main()
File "/Applications/PyCharm.app/Contents/helpers/pydev/pydevd.py", line 1752, in main
globals = debugger.run(setup['file'], None, None, is_module)
File "/Applications/PyCharm.app/Contents/helpers/pydev/pydevd.py", line 1147, in run
pydev_imports.execfile(file, globals, locals) # execute the script
File "/Applications/PyCharm.app/Contents/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "/git/huddl/python/hamspam/enron.py", line 205, in <module>
client = Client()
File "/usr/local/lib/python3.7/site-packages/distributed/client.py", line 712, in __init__
self.start(timeout=timeout)
File "/usr/local/lib/python3.7/site-packages/distributed/client.py", line 858, in start
sync(self.loop, self._start, **kwargs)
File "/usr/local/lib/python3.7/site-packages/distributed/utils.py", line 331, in sync
six.reraise(*error[0])
File "/usr/local/lib/python3.7/site-packages/six.py", line 693, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/distributed/utils.py", line 316, in f
result[0] = yield future
File "/usr/local/lib/python3.7/site-packages/tornado/gen.py", line 729, in run
value = future.result()
File "/usr/local/lib/python3.7/site-packages/tornado/gen.py", line 736, in run
yielded = self.gen.throw(*exc_info) # type: ignore
File "/usr/local/lib/python3.7/site-packages/distributed/client.py", line 928, in _start
yield self.cluster
File "/usr/local/lib/python3.7/site-packages/tornado/gen.py", line 729, in run
value = future.result()
File "/usr/local/Cellar/python/3.7.3/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/tasks.py", line 603, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/usr/local/lib/python3.7/site-packages/tornado/gen.py", line 736, in run
yielded = self.gen.throw(*exc_info) # type: ignore
File "/usr/local/lib/python3.7/site-packages/distributed/deploy/local.py", line 284, in _start
yield [self._start_worker(**self.worker_kwargs) for i in range(n_workers)]
File "/usr/local/lib/python3.7/site-packages/tornado/gen.py", line 729, in run
value = future.result()
File "/usr/local/lib/python3.7/site-packages/tornado/gen.py", line 501, in callback
result_list.append(f.result())
File "/usr/local/lib/python3.7/site-packages/tornado/gen.py", line 742, in run
yielded = self.gen.send(value)
File "/usr/local/lib/python3.7/site-packages/distributed/deploy/local.py", line 316, in _start_worker
raise gen.TimeoutError("Worker failed to start")
tornado.util.TimeoutError: Worker failed to start
Any advice on this?
There will be even more issues/complications when trying to use a LocalCluster mode -but that will be saved for a different question.

Categories

Resources