I randomly get the following errors (i.e most of the times the scripts work fine)
File ".../backend/case_status_consumer.py", line 28, in <module>
channel.start_consuming()
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/blocking_connection.py", line 955, in start_consuming
self.connection.process_data_events()
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/blocking_connection.py", line 240, in process_data_events
if self._handle_read():
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/blocking_connection.py", line 348, in _handle_read
super(BlockingConnection, self)._handle_read()
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/base_connection.py", line 351, in _handle_read
self._on_data_available(data)
File "/usr/local/lib/python2.7/dist-packages/pika/connection.py", line 1285, in _on_data_available
self._process_frame(frame_value)
File "/usr/local/lib/python2.7/dist-packages/pika/connection.py", line 1365, in _process_frame
self._deliver_frame_to_channel(frame_value)
File "/usr/local/lib/python2.7/dist-packages/pika/connection.py", line 976, in _deliver_frame_to_channel
return self._channels[value.channel_number]._handle_content_frame(value)
File "/usr/local/lib/python2.7/dist-packages/pika/channel.py", line 792, in _handle_content_frame
self._on_deliver(*response)
File "/usr/local/lib/python2.7/dist-packages/pika/channel.py", line 886, in _on_deliver
body)
File ".../backend/case_status_consumer.py", line 14, in consume_case_status
case_num['case_year'])
File ".../backend/src/fetcher/case_update.py", line 132, in case_update
properties=pika.BasicProperties(delivery_mode=2))
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/blocking_connection.py", line 572, in basic_publish
(properties, body), False)
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/blocking_connection.py", line 1159, in _send_method
self.connection.send_method(self.channel_number, method_frame, content)
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/blocking_connection.py", line 274, in send_method
self._send_method(channel_number, method_frame, content)
File "/usr/local/lib/python2.7/dist-packages/pika/connection.py", line 1503, in _send_method
self._send_frame(frame.Method(channel_number, method_frame))
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/blocking_connection.py", line 417, in _send_frame
super(BlockingConnection, self)._send_frame(frame_value)
File "/usr/local/lib/python2.7/dist-packages/pika/connection.py", line 1490, in _send_frame
self._flush_outbound()
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/blocking_connection.py", line 377, in _flush_outbound
if self._handle_write():
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/base_connection.py", line 365, in _handle_write
return self._handle_error(error)
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/base_connection.py", line 302, in _handle_error
self._handle_disconnect()
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/base_connection.py", line 248, in _handle_disconnect
self._adapter_disconnect()
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/blocking_connection.py", line 318, in _adapter_disconnect
self._check_state_on_disconnect()
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/blocking_connection.py", line 371, in _check_state_on_disconnect
raise exceptions.ConnectionClosed()
pika.exceptions.ConnectionClosed
And I get a similar error while producing too.
File ".../backend/check_for_orders.py", line 115, in <module>
dated=order_dated, ors_fetch=False)
File ".../backend/src/fetcher/case_update.py", line 132, in case_update
properties=pika.BasicProperties(delivery_mode=2))
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/blocking_connection.py", line 572, in basic_publish
(properties, body), False)
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/blocking_connection.py", line 1159, in _send_method
self.connection.send_method(self.channel_number, method_frame, content)
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/blocking_connection.py", line 274, in send_method
self._send_method(channel_number, method_frame, content)
File "/usr/local/lib/python2.7/dist-packages/pika/connection.py", line 1503, in _send_method
self._send_frame(frame.Method(channel_number, method_frame))
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/blocking_connection.py", line 417, in _send_frame
super(BlockingConnection, self)._send_frame(frame_value)
File "/usr/local/lib/python2.7/dist-packages/pika/connection.py", line 1490, in _send_frame
self._flush_outbound()
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/blocking_connection.py", line 377, in _flush_outbound
if self._handle_write():
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/base_connection.py", line 365, in _handle_write
return self._handle_error(error)
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/base_connection.py", line 302, in _handle_error
self._handle_disconnect()
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/base_connection.py", line 248, in _handle_disconnect
self._adapter_disconnect()
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/blocking_connection.py", line 318, in _adapter_disconnect
self._check_state_on_disconnect()
File "/usr/local/lib/python2.7/dist-packages/pika/adapters/blocking_connection.py", line 371, in _check_state_on_disconnect
raise exceptions.ConnectionClosed()
pika.exceptions.ConnectionClosed
My connection is set to:
connection = pika.BlockingConnection(pika.ConnectionParameters('localhost', heartbeat_interval=0))
I have even tried setting the heartbeat interval to a longer time (like 60 and 90), but I still get these errors.
I cannot post the full code but what my scripts are doing is, using 'requests' to fetch a page (with timeout as 50 seconds and maximum 4 retries)
This issue is fixed in pika 0.10.0
Related
Traceback (most recent call last):
File "C:\Users\HP\myproject\lib\site-packages\pip_vendor\urllib3\response.py", line 438, in _error_catcher
yield
File "C:\Users\HP\myproject\lib\site-packages\pip_vendor\urllib3\response.py", line 519, in read
data = self._fp.read(amt) if not fp_closed else b""
File "C:\Users\HP\myproject\lib\site-packages\pip_vendor\cachecontrol\filewrapper.py", line 62, in read
data = self.__fp.read(amt)
File "C:\Users\HP\AppData\Local\Programs\Python\Python310\lib\http\client.py", line 465, in read
s = self.fp.read(amt)
File "C:\Users\HP\AppData\Local\Programs\Python\Python310\lib\socket.py", line 705, in readinto
return self._sock.recv_into(b)
File "C:\Users\HP\AppData\Local\Programs\Python\Python310\lib\ssl.py", line 1273, in recv_into
return self.read(nbytes, buffer)
File "C:\Users\HP\AppData\Local\Programs\Python\Python310\lib\ssl.py", line 1129, in read
return self._sslobj.read(len, buffer)
TimeoutError: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\cli\base_command.py", line 173, in _main
status = self.run(options, args)
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\cli\req_command.py", line 203, in wrapper
return func(self, options, args)
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\commands\install.py", line 315, in run
requirement_set = resolver.resolve(
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\resolution\resolvelib\resolver.py", line 94, in resolve
result = self._result = resolver.resolve(
File "C:\Users\HP\myproject\lib\site-packages\pip_vendor\resolvelib\resolvers.py", line 472, in resolve
state = resolution.resolve(requirements, max_rounds=max_rounds)
File "C:\Users\HP\myproject\lib\site-packages\pip_vendor\resolvelib\resolvers.py", line 341, in resolve
self._add_to_criteria(self.state.criteria, r, parent=None)
File "C:\Users\HP\myproject\lib\site-packages\pip_vendor\resolvelib\resolvers.py", line 172, in _add_to_criteria
if not criterion.candidates:
File "C:\Users\HP\myproject\lib\site-packages\pip_vendor\resolvelib\structs.py", line 151, in bool
return bool(self._sequence)
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\resolution\resolvelib\found_candidates.py", line 140, in bool
return any(self)
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\resolution\resolvelib\found_candidates.py", line 128, in
return (c for c in iterator if id(c) not in self._incompatible_ids)
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\resolution\resolvelib\found_candidates.py", line 32, in _iter_built
candidate = func()
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\resolution\resolvelib\factory.py", line 204, in _make_candidate_from_link
self._link_candidate_cache[link] = LinkCandidate(
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\resolution\resolvelib\candidates.py", line 295, in init
super().init(
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\resolution\resolvelib\candidates.py", line 156, in init
self.dist = self._prepare()
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\resolution\resolvelib\candidates.py", line 227, in _prepare
dist = self._prepare_distribution()
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\resolution\resolvelib\candidates.py", line 305, in _prepare_distribution
return self._factory.preparer.prepare_linked_requirement(
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\operations\prepare.py", line 508, in prepare_linked_requirement
return self._prepare_linked_requirement(req, parallel_builds)
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\operations\prepare.py", line 550, in _prepare_linked_requirement
local_file = unpack_url(
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\operations\prepare.py", line 239, in unpack_url
file = get_http_url(
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\operations\prepare.py", line 102, in get_http_url
from_path, content_type = download(link, temp_dir.path)
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\network\download.py", line 145, in call
for chunk in chunks:
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\cli\progress_bars.py", line 144, in iter
for x in it:
File "C:\Users\HP\myproject\lib\site-packages\pip_internal\network\utils.py", line 63, in response_chunks
for chunk in response.raw.stream(
File "C:\Users\HP\myproject\lib\site-packages\pip_vendor\urllib3\response.py", line 576, in stream
data = self.read(amt=amt, decode_content=decode_content)
File "C:\Users\HP\myproject\lib\site-packages\pip_vendor\urllib3\response.py", line 512, in read
with self._error_catcher():
File "C:\Users\HP\AppData\Local\Programs\Python\Python310\lib\contextlib.py", line 153, in exit
self.gen.throw(typ, value, traceback)
File "C:\Users\HP\myproject\lib\site-packages\pip_vendor\urllib3\response.py", line 443, in _error_catcher
raise ReadTimeoutError(self._pool, None, "Read timed out.")
pip._vendor.urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='files.pythonhosted.org', port=443): Read timed out.
Test the Django installation in a virtual environment,
in cmd:
> python -m venv env
> cd env/Scripts/activate
> pip install django
I'm writing a Dataflow pipeline using Apache beam to add large batches of rows of data to Bigtable.
apache-beam==2.24.0
google-cloud-bigtable==2.4.0
I have the following method used in my pipeline to create the Bigtable row(s) prior to writing to Bigtable:
class CreateBigtableRow(beam.DoFn):
def __init__(self, settings):
self.column_family = settings["bigtable_column_family"]
super(CreateBigtableRow, self).__init__()
def process(self, usage_data, *args, **kwargs):
row_key = BigTable.generate_row_key(usage_data, key_order)
return [
BigTable.create_row_and_assign_values(
row_key, usage_data, self.column_family
)
]
where `create_row_and_assign_values is defined as:
def create_row_and_assign_values(
cls, key: str, row: dict, column_family: str
) -> DirectRow:
table_row = DirectRow(key.encode())
for key, val in row.items():
if isinstance(val, float):
val = struct.pack(">d", val)
table_row.set_cell(column_family, key.encode(), val)
return table_row
My pipeline is as follows:
with beam.Pipeline(options=pipeline_options) as pipe:
(
pipe
| beam.Create(["/sample_files/*combined*"]) # reads sample csv file
| fileio.MatchAll()
| fileio.ReadMatches()
| beam.FlatMap(
lambda file: csv.DictReader(open(file.metadata.path))
)
| "Transform to Usage dict" >> beam.ParDo(TransformToBigtableData())
| "Create Bigtable Row" >> beam.ParDo(CreateBigtableRow(bigtable_settings))
| WriteToBigTable(
project_id=bigtable_settings["bigtable_project"],
instance_id=bigtable_settings["bigtable_instance"],
table_id=bigtable_settings["bigtable_table"])
)
The problem I'm having is I get the error
_pickle.PicklingError: Can't pickle <class 'Mutation'>: attribute lookup Mutation on __main__ failed [while running 'Create Bigtable Row']
when running the pipeline. I've added steps to manually batch process the records by using the google-cloud-bigtable library's Bigtable Client, but would prefer to use the build-in WriteToBigTable method as it handles everything for me.
Full stack trace:
Traceback (most recent call last):
File "/app/src/ingest/main.py", line 226, in <module>
run(
File "/app/src/ingest/main.py", line 149, in run
(
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/pipeline.py", line 596, in __exit__
self.result = self.run()
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/pipeline.py", line 546, in run
return Pipeline.from_runner_api(
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/pipeline.py", line 573, in run
return self.runner.run_pipeline(self, self._options)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/direct/direct_runner.py", line 131, in run_pipeline
return runner.run_pipeline(pipeline, options)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 195, in run_pipeline
self._latest_run_result = self.run_via_runner_api(
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 206, in run_via_runner_api
return self.run_stages(stage_context, stages)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 384, in run_stages
stage_results = self._run_stage(
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 646, in _run_stage
self._run_bundle(
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 769, in _run_bundle
result, splits = bundle_manager.process_bundle(
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 1080, in process_bundle
result_future = self._worker_handler.control_conn.push(process_bundle_req)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py", line 378, in push
response = self.worker.do_instruction(request)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/sdk_worker.py", line 597, in do_instruction
return getattr(self, request_type)(
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/sdk_worker.py", line 635, in process_bundle
bundle_processor.process_bundle(instruction_id))
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/bundle_processor.py", line 995, in process_bundle
input_op_by_transform_id[element.transform_id].process_encoded(
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/bundle_processor.py", line 221, in process_encoded
self.output(decoded_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/operations.py", line 354, in output
cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/operations.py", line 216, in receive
self.consumer.process(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/operations.py", line 714, in process
delayed_applications = self.dofn_runner.process(o)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1235, in process
self._reraise_augmented(exn)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1233, in process
return self.do_fn_invoker.invoke_process(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 571, in invoke_process
self.output_processor.process_outputs(
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1396, in process_outputs
self.main_receivers.receive(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/operations.py", line 216, in receive
self.consumer.process(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/operations.py", line 714, in process
delayed_applications = self.dofn_runner.process(o)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1235, in process
self._reraise_augmented(exn)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1233, in process
return self.do_fn_invoker.invoke_process(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 571, in invoke_process
self.output_processor.process_outputs(
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1396, in process_outputs
self.main_receivers.receive(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/operations.py", line 216, in receive
self.consumer.process(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/operations.py", line 714, in process
delayed_applications = self.dofn_runner.process(o)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1235, in process
self._reraise_augmented(exn)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1233, in process
return self.do_fn_invoker.invoke_process(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 571, in invoke_process
self.output_processor.process_outputs(
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1396, in process_outputs
self.main_receivers.receive(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/operations.py", line 216, in receive
self.consumer.process(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/operations.py", line 714, in process
delayed_applications = self.dofn_runner.process(o)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1235, in process
self._reraise_augmented(exn)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1233, in process
return self.do_fn_invoker.invoke_process(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 571, in invoke_process
self.output_processor.process_outputs(
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1396, in process_outputs
self.main_receivers.receive(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/operations.py", line 216, in receive
self.consumer.process(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/operations.py", line 714, in process
delayed_applications = self.dofn_runner.process(o)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1235, in process
self._reraise_augmented(exn)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1233, in process
return self.do_fn_invoker.invoke_process(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 571, in invoke_process
self.output_processor.process_outputs(
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1396, in process_outputs
self.main_receivers.receive(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/operations.py", line 216, in receive
self.consumer.process(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/operations.py", line 714, in process
delayed_applications = self.dofn_runner.process(o)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1235, in process
self._reraise_augmented(exn)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1233, in process
return self.do_fn_invoker.invoke_process(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 571, in invoke_process
self.output_processor.process_outputs(
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1396, in process_outputs
self.main_receivers.receive(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/operations.py", line 216, in receive
self.consumer.process(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/operations.py", line 714, in process
delayed_applications = self.dofn_runner.process(o)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1235, in process
self._reraise_augmented(exn)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1316, in _reraise_augmented
raise new_exn.with_traceback(tb)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1233, in process
return self.do_fn_invoker.invoke_process(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 571, in invoke_process
self.output_processor.process_outputs(
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/common.py", line 1396, in process_outputs
self.main_receivers.receive(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/operations.py", line 215, in receive
self.update_counters_start(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/operations.py", line 179, in update_counters_start
self.opcounter.update_from(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/opcounters.py", line 211, in update_from
self.do_sample(windowed_value)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/runners/worker/opcounters.py", line 250, in do_sample
self.coder_impl.get_estimated_size_and_observables(windowed_value))
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/coders/coder_impl.py", line 1371, in get_estimated_size_and_observables
self._value_coder.get_estimated_size_and_observables(
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/coders/coder_impl.py", line 358, in get_estimated_size_and_observables
self.encode_to_stream(value, out, nested)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/coders/coder_impl.py", line 422, in encode_to_stream
self.fallback_coder_impl.encode_to_stream(value, stream, nested)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/coders/coder_impl.py", line 262, in encode_to_stream
return stream.write(self._encoder(value), nested)
File "/opt/pysetup/.venv/lib/python3.9/site-packages/apache_beam/coders/coders.py", line 800, in <lambda>
lambda x: dumps(x, protocol), pickle.loads)
_pickle.PicklingError: Can't pickle <class 'Mutation'>: attribute lookup Mutation on __main__ failed [while running 'Create Bigtable Row']
Your google-cloud-bigtable version is too high.
There is some movement in updating apache-beam dependencies here
They have the same issue. Can you roll back your bigtable version to something before 2? If you run this:
pip install apache-beam[gcp]
It'll install the recommended version.
i am trying to install scipy package on one of my Pycharm projects. i get the following Attribute Error,
AttributeError: 'xml.etree.ElementTree.Element' object has no attribute 'text_temp'
following is the exception that i am getting,
Collecting scipy
Exception:
Traceback (most recent call last):
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_internal/cli/base_command.py", line 179, in main
status = self.run(options, args)
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_internal/commands/install.py", line 315, in run
resolver.resolve(requirement_set)
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_internal/resolve.py", line 131, in resolve
self._resolve_one(requirement_set, req)
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_internal/resolve.py", line 294, in _resolve_one
abstract_dist = self._get_abstract_dist_for(req_to_install)
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_internal/resolve.py", line 242, in _get_abstract_dist_for
self.require_hashes
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_internal/operations/prepare.py", line 269, in prepare_linked_requirement
req.populate_link(finder, upgrade_allowed, require_hashes)
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_internal/req/req_install.py", line 196, in populate_link
self.link = finder.find_requirement(self, upgrade)
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_internal/index.py", line 639, in find_requirement
all_candidates = self.find_all_candidates(req.name)
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_internal/index.py", line 614, in find_all_candidates
self._package_versions(page.iter_links(), search)
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_internal/index.py", line 775, in _package_versions
for link in self._sort_links(links):
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_internal/index.py", line 759, in _sort_links
for link in links:
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_internal/index.py", line 971, in iter_links
namespaceHTMLElements=False,
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_vendor/html5lib/html5parser.py", line 47, in parse
return p.parse(doc, **kwargs)
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_vendor/html5lib/html5parser.py", line 289, in parse
self._parse(stream, False, None, *args, **kwargs)
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_vendor/html5lib/html5parser.py", line 134, in _parse
self.mainLoop()
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_vendor/html5lib/html5parser.py", line 239, in mainLoop
new_token = phase.processSpaceCharacters(new_token)
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_vendor/html5lib/html5parser.py", line 469, in processSpaceCharacters
self.tree.insertText(token["data"])
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_vendor/html5lib/treebuilders/base.py", line 357, in insertText
parent.insertText(data)
File "/Users/rohitgite/PycharmProjects/HelloWorld/venv/lib/python3.7/site-packages/pip-19.0.3-py3.7.egg/pip/_vendor/html5lib/treebuilders/etree.py", line 109, in insertText
if not self._element.text_temp:
AttributeError: 'xml.etree.ElementTree.Element' object has no attribute 'text_temp'
I am getting the following error when I try to start the server locally (./manage.py runserver).
Posting the question after spending 5 hours of searching solution.
Attached the error stack. I am using Django 2.2.2.
Watching for file changes with StatReloader
Performing system checks...
Traceback (most recent call last):
File "./manage.py", line 26, in <module>
execute_from_command_line(sys.argv)
File "/home/santhosh/Desktop/working_dir/piccolo/venv/lib/python3.5/site-packages/django/core/management/__init__.py", line 381, in execute_from_command_line
utility.execute()
File "/home/santhosh/Desktop/working_dir/piccolo/venv/lib/python3.5/site-packages/django/core/management/__init__.py", line 375, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "/home/santhosh/Desktop/working_dir/piccolo/venv/lib/python3.5/site-packages/django/core/management/base.py", line 323, in run_from_argv
self.execute(*args, **cmd_options)
File "/home/santhosh/Desktop/working_dir/piccolo/venv/lib/python3.5/site-packages/django/core/management/commands/runserver.py", line 60, in execute
super().execute(*args, **options)
File "/home/santhosh/Desktop/working_dir/piccolo/venv/lib/python3.5/site-packages/django/core/management/base.py", line 364, in execute
output = self.handle(*args, **options)
File "/home/santhosh/Desktop/working_dir/piccolo/venv/lib/python3.5/site-packages/django/core/management/commands/runserver.py", line 95, in handle
self.run(**options)
File "/home/santhosh/Desktop/working_dir/piccolo/venv/lib/python3.5/site-packages/django/core/management/commands/runserver.py", line 102, in run
autoreload.run_with_reloader(self.inner_run, **options)
File "/home/santhosh/Desktop/working_dir/piccolo/venv/lib/python3.5/site-packages/django/utils/autoreload.py", line 585, in run_with_reloader
start_django(reloader, main_func, *args, **kwargs)
File "/home/santhosh/Desktop/working_dir/piccolo/venv/lib/python3.5/site-packages/django/utils/autoreload.py", line 570, in start_django
reloader.run(django_main_thread)
File "/home/santhosh/Desktop/working_dir/piccolo/venv/lib/python3.5/site-packages/django/utils/autoreload.py", line 288, in run
self.run_loop()
File "/home/santhosh/Desktop/working_dir/piccolo/venv/lib/python3.5/site-packages/django/utils/autoreload.py", line 294, in run_loop
next(ticker)
File "/home/santhosh/Desktop/working_dir/piccolo/venv/lib/python3.5/site-packages/django/utils/autoreload.py", line 334, in tick
for filepath, mtime in self.snapshot_files():
File "/home/santhosh/Desktop/working_dir/piccolo/venv/lib/python3.5/site-packages/django/utils/autoreload.py", line 350, in snapshot_files
for file in self.watched_files():
File "/home/santhosh/Desktop/working_dir/piccolo/venv/lib/python3.5/site-packages/django/utils/autoreload.py", line 249, in watched_files
yield from iter_all_python_module_files()
File "/home/santhosh/Desktop/working_dir/piccolo/venv/lib/python3.5/site-packages/django/utils/autoreload.py", line 103, in iter_all_python_module_files
return iter_modules_and_files(modules, frozenset(_error_files))
File "/home/santhosh/Desktop/working_dir/piccolo/venv/lib/python3.5/site-packages/django/utils/autoreload.py", line 116, in iter_modules_and_files
if module.__name__ == '__main__':
File "/usr/lib/python3/dist-packages/py/_apipkg.py", line 171, in __getattribute__
return getattr(getmod(), name)
File "/usr/lib/python3/dist-packages/py/_error.py", line 43, in __getattr__
raise AttributeError(name)
AttributeError: __name__
Thanks in advance.
With the following codes under beam 2.14.0
| "FixedWindow" >> beam.WindowInto(beam.window.FixedWindows(4 * 60),
trigger=beam.trigger.Repeatedly(
beam.trigger.AfterProcessingTime(delay=1 * 60)
),
accumulation_mode=beam.trigger.AccumulationMode.DISCARDING)
The following error comes up
Traceback (most recent call last):
File "beam_home.py", line 287, in <module>
run()
File "beam_home.py", line 282, in run
p.run().wait_until_finish()
File "/usr/local/lib/python2.7/site-packages/apache_beam/pipeline.py", line 406, in run
self._options).run(False)
File "/usr/local/lib/python2.7/site-packages/apache_beam/pipeline.py", line 419, in run
return self.runner.run_pipeline(self, self._options)
File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/direct/direct_runner.py", line 128, in run_pipeline
return runner.run_pipeline(pipeline, options)
File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/portability/fn_api_runner.py", line 294, in run_pipeline
default_environment=self._default_environment))
File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/portability/fn_api_runner.py", line 301, in run_via_runner_api
return self.run_stages(stage_context, stages)
File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/portability/fn_api_runner.py", line 383, in run_stages
stage_context.safe_coders)
File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/portability/fn_api_runner.py", line 655, in _run_stage
result, splits = bundle_manager.process_bundle(data_input, data_output)
File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/portability/fn_api_runner.py", line 1460, in process_bundle
process_bundle_id, transform_id, elements)
File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/portability/fn_api_runner.py", line 1356, in _send_input_to_worker
for byte_stream in byte_streams:
File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/portability/fn_api_runner.py", line 186, in __iter__
for wkvs in windowed_key_values(key, windowed_values):
File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/trigger.py", line 966, in process_entire_key
state, windowed_values, output_watermark):
File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/trigger.py", line 1130, in process_elements
self.trigger_fn.on_element(value, window, context)
File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/trigger.py", line 515, in on_element
self.underlying.on_element(element, window, context)
File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/trigger.py", line 373, in on_element
self.early.on_element(element, window, NestedContext(context, 'early'))
File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/trigger.py", line 515, in on_element
self.underlying.on_element(element, window, context)
File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/trigger.py", line 307, in on_element
'', TimeDomain.REAL_TIME, context.get_current_time() + self.delay)
File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/trigger.py", line 759, in get_current_time
return self._outer.get_current_time()
File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/trigger.py", line 733, in get_current_time
return self._clock.time()
AttributeError: 'NoneType' object has no attribute 'time'
Anything am I missing?
It doesn't look like you are missing anything.
It's apparently a known issue. Please see BEAM-5132.
I think the best workaround is to avoid using AfterProcessingTime which it the root cause. It's annoying but you can imitate its effect in a ParDo class.