I'm doing a pipeline in Azure ML SDK. After I had run the pipeline for some amount of times it reported I had reached the Snapshot limit of 300MB. I followed some of the fixes that was proposed:
Each step script is moved to a separate subfolder
I added a datastore to the pipeline
This line was added: azureml._restclient.snapshots_client.SNAPSHOT_MAX_SIZE_BYTES = 1000
But then a new Snapshot error occurred after I submitted my pipeline:
pipeline1 = Pipeline(default_source_directory=".", default_datastore=def_blob_store, workspace=ws, steps=[prep_step, hd_step, register_model_step])
THE ERROR MESSAGE:
WARNING:root:If 'script' has been provided here and a script file name has been specified in 'run_config', 'script' provided in ScriptRunConfig initialization will take precedence.
---------------------------------------------------------------------------
SnapshotException Traceback (most recent call last)
<ipython-input-14-05c5aa4991aa> in <module>
----> 1 pipeline1 = Pipeline(default_source_directory=".", default_datastore=def_blob_store, workspace=ws, steps=[prep_step, hd_step, register_model_step])
2 pipeline1.validate()
3 pipeline_run = Experiment(ws, 'health_insuarance').submit(pipeline1, regenerate_outputs=False)
4 RunDetails(pipeline_run).show()
/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/core/_experiment_method.py in wrapper(self, *args, **kwargs)
95 """
96 ExperimentSubmitRegistrar.register_submit_function(self.__class__, submit_function)
---> 97 return init_func(self, *args, **kwargs)
98 return wrapper
99 return real_decorator
/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/pipeline/core/pipeline.py in __init__(self, workspace, steps, description, default_datastore, default_source_directory, resolve_closure, _workflow_provider, _service_endpoint, **kwargs)
175 raise ValueError('parameter %s is not recognized for Pipeline ' % key)
176 self._enable_email_notification = enable_email_notification
--> 177 self._graph = self._graph_builder.build(self._name, steps, finalize=False)
178
179 def _set_experiment_name(self, name):
/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/pipeline/core/builder.py in build(self, name, steps, finalize, regenerate_outputs)
1479 pass
1480
-> 1481 graph = self.construct(name, steps)
1482 if finalize:
1483 graph.finalize(regenerate_outputs=regenerate_outputs)
/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/pipeline/core/builder.py in construct(self, name, steps)
1501 self._graph = Graph(name, self._context)
1502 self._nodeStack.append([])
-> 1503 self.process_collection(steps)
1504 for builder in self._builderStack[::-1]:
1505 builder.apply_rules()
/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/pipeline/core/builder.py in process_collection(self, collection)
1537 self._nodeStack.append([])
1538 self._builderStack.append(builder)
-> 1539 builder.process_collection(collection)
1540 added_nodes = self._nodeStack.pop()
1541 self._nodeStack[-1].extend(added_nodes)
/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/pipeline/core/builder.py in process_collection(self, collection)
1828 """
1829 for item in collection:
-> 1830 self._base_builder.process_collection(item)
1831
1832 def apply_rules(self):
/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/pipeline/core/builder.py in process_collection(self, collection)
1531 # just a step?
1532 if isinstance(collection, PipelineStep):
-> 1533 return self.process_step(collection)
1534
1535 # delegate to correct builder
/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/pipeline/core/builder.py in process_step(self, step)
1575 return self._step2node[step]
1576
-> 1577 node = step.create_node(self._graph, self._default_datastore, self._context)
1578 self.assert_node_valid(step, self._graph, node)
1579
/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/pipeline/steps/hyper_drive_step.py in create_node(self, graph, default_datastore, context)
247 """
248 hyperdrive_config, reuse_hashable_config = self._get_hyperdrive_config(context._workspace,
--> 249 context._experiment_name)
250 self._params[HyperDriveStep._run_config_param_name] = json.dumps(hyperdrive_config)
251 self._params[HyperDriveStep._run_reuse_hashable_config] = json.dumps(reuse_hashable_config)
/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/pipeline/steps/hyper_drive_step.py in _get_hyperdrive_config(self, workspace, experiment_name)
323
324 hyperdrive_dto = _search._create_experiment_dto(self._hyperdrive_config, workspace,
--> 325 experiment_name, telemetry_values)
326
327 hyperdrive_config = hyperdrive_dto.as_dict()
/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/train/hyperdrive/_search.py in _create_experiment_dto(hyperdrive_config, workspace, experiment_name, telemetry_values, activity_logger, **kwargs)
41 if hyperdrive_config.source_directory is not None:
42 snapshot_client = SnapshotsClient(workspace.service_context)
---> 43 snapshot_id = snapshot_client.create_snapshot(hyperdrive_config.source_directory)
44
45 if activity_logger is not None:
/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/_restclient/snapshots_client.py in create_snapshot(self, file_or_folder_path, retry_on_failure, raise_on_validation_failure)
83 exclude_function = ignore_file.is_file_excluded
84
---> 85 self._validate_snapshot_size(file_or_folder_path, exclude_function, raise_on_validation_failure)
86
87 # Get the previous snapshot for this project
/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/_restclient/snapshots_client.py in _validate_snapshot_size(self, file_or_folder_path, exclude_function, raise_on_validation_failure)
61 "\n".format(file_or_folder_path, SNAPSHOT_MAX_SIZE_BYTES / ONE_MB)
62 if raise_on_validation_failure:
---> 63 raise SnapshotException(error_message)
64 else:
65 self._logger.warning(error_message)
SnapshotException: SnapshotException:
Message: ====================================================================
While attempting to take snapshot of ./train/
Your total snapshot size exceeds the limit of 0.00095367431640625 MB.
Please see http://aka.ms/aml-largefiles on how to work with large files.
====================================================================
InnerException None
ErrorResponse
{
"error": {
"message": "====================================================================\n\nWhile attempting to take snapshot of ./train/\nYour total snapshot size exceeds the limit of 0.00095367431640625 MB.\nPlease see http://aka.ms/aml-largefiles on how to work with large files.\n\n====================================================================\n\n"
}
}
Any idea how I fix this?
The fulls script is here: Script at Github
Alright, so I found the fix.
I changed this line by adding a number equvilant to 1GB: azureml._restclient.snapshots_client.SNAPSHOT_MAX_SIZE_BYTES = 1000000000
For some reason, you have to define the size in BYTES and not megabytes even though the default is 300 MB. Not especially intuitive.
Related
Trying to Access a Shared Folder using the following code :
credentials = Credentials(username = user_name, password = "secret")
config = Configuration(server ='outlook.office365.com', credentials = credentials, auth_type=NTLM)
account = Account(primary_smtp_address = 'shared_mail#domain.com', credentials = credentials, autodiscover = False, config = config, access_type = DELEGATE,)
The above three lines of Code work perfectly but we are unable to get the root,
the following code : account.root.tree() or account.root throws the following error:
KeyError Traceback (most recent call last)
~\anaconda3\lib\site-packages\cached_property.py in __get__(self, obj, cls)
68 # check if the value was computed before the lock was acquired
---> 69 return obj_dict[name]
70
KeyError: 'root'
During handling of the above exception, another exception occurred:
ErrorNonExistentMailbox Traceback (most recent call last)
<ipython-input-46-a90a4f76ca21> in <module>
2 logging.basicConfig(level=logging.DEBUG)
3
----> 4 account.root.tree()
~\anaconda3\lib\site-packages\cached_property.py in __get__(self, obj, cls)
71 except KeyError:
72 # if not, do the calculation and release the lock
---> 73 return obj_dict.setdefault(name, self.func(obj))
74
75
~\anaconda3\lib\site-packages\exchangelib\account.py in root(self)
268 #threaded_cached_property
269 def root(self):
--> 270 return Root.get_distinguished(account=self)
271
272 #threaded_cached_property
~\anaconda3\lib\site-packages\exchangelib\folders\roots.py in get_distinguished(cls, account)
107 return cls.resolve(
108 account=account,
--> 109 folder=cls(account=account, name=cls.DISTINGUISHED_FOLDER_ID, is_distinguished=True)
110 )
111 except ErrorFolderNotFound:
~\anaconda3\lib\site-packages\exchangelib\folders\base.py in resolve(cls, account, folder)
485 def resolve(cls, account, folder):
486 # Resolve a single folder
--> 487 folders = list(FolderCollection(account=account, folders=[folder]).resolve())
488 if not folders:
489 raise ErrorFolderNotFound('Could not find folder %r' % folder)
~\anaconda3\lib\site-packages\exchangelib\folders\collections.py in resolve(self)
254 additional_fields = self.get_folder_fields(target_cls=self._get_target_cls(), is_complex=None)
255 for f in self.__class__(account=self.account, folders=resolveable_folders).get_folders(
--> 256 additional_fields=additional_fields
257 ):
258 yield f
~\anaconda3\lib\site-packages\exchangelib\folders\collections.py in get_folders(self, additional_fields)
317 folders=self.folders,
318 additional_fields=additional_fields,
--> 319 shape=ID_ONLY,
320 ):
321 yield f
~\anaconda3\lib\site-packages\exchangelib\services\get_folder.py in call(self, folders, additional_fields, shape)
32 **dict(
33 additional_fields=additional_fields,
---> 34 shape=shape,
35 )
36 )):
~\anaconda3\lib\site-packages\exchangelib\services\common.py in _pool_requests(self, payload_func, items, **kwargs)
538 for i, chunk in enumerate(chunkify(items, self.chunk_size), start=1):
539 log.debug('Processing %s chunk %s containing %s items', self.__class__.__name__, i, len(chunk))
--> 540 for elem in self._get_elements(payload=payload_func(chunk, **kwargs)):
541 yield elem
542
~\anaconda3\lib\site-packages\exchangelib\services\common.py in _get_elements_in_response(self, response)
401 def _get_elements_in_response(self, response):
402 for msg in response:
--> 403 container_or_exc = self._get_element_container(message=msg, name=self.element_container_name)
404 if isinstance(container_or_exc, (bool, Exception)):
405 yield container_or_exc
~\anaconda3\lib\site-packages\exchangelib\services\common.py in _get_element_container(self, message, response_message, name)
360 # rspclass == 'Error', or 'Success' and not 'NoError'
361 try:
--> 362 raise self._get_exception(code=response_code, text=msg_text, msg_xml=msg_xml)
363 except self.ERRORS_TO_CATCH_IN_RESPONSE as e:
364 return e
ErrorNonExistentMailbox: Mailbox does not exist.
The same code seems to be working here : https://medium.com/#theamazingexposure/accessing-shared-mailbox-using-exchangelib-python-f020e71a96ab
Also checked this thread https://github.com/ecederstrand/exchangelib/issues/391 and tried almost all the solutions but facing the same error.
I was working just fine with Google Colaboratory and suddenly this error started to pop up each time I try to load any type of file. It first started when I was trying to read an hdf file, now everything won't open.
---------------------------------------------------------------------------
OSError Traceback (most recent call last)
<ipython-input-8-65c4e8d1c435> in <module>()
----> 1 eo=EOPatch.load('./20.Clean_Textural_features/eopatch_0')
2 eo
9 frames
/usr/local/lib/python3.6/dist-packages/eolearn/core/eodata.py in load(path, features, lazy_loading, filesystem)
530 path = '/'
531
--> 532 return load_eopatch(EOPatch(), filesystem, path, features=features, lazy_loading=lazy_loading)
533
534 def time_series(self, ref_date=None, scale_time=1):
/usr/local/lib/python3.6/dist-packages/eolearn/core/eodata_io.py in load_eopatch(eopatch, filesystem, patch_location, features, lazy_loading)
76 loading_data = executor.map(lambda loader: loader.load(), loading_data)
77
---> 78 for (ftype, fname, _), value in zip(features, loading_data):
79 eopatch[(ftype, fname)] = value
80
/usr/lib/python3.6/concurrent/futures/_base.py in result_iterator()
584 # Careful not to keep a reference to the popped future
585 if timeout is None:
--> 586 yield fs.pop().result()
587 else:
588 yield fs.pop().result(end_time - time.monotonic())
/usr/lib/python3.6/concurrent/futures/_base.py in result(self, timeout)
423 raise CancelledError()
424 elif self._state == FINISHED:
--> 425 return self.__get_result()
426
427 self._condition.wait(timeout)
/usr/lib/python3.6/concurrent/futures/_base.py in __get_result(self)
382 def __get_result(self):
383 if self._exception:
--> 384 raise self._exception
385 else:
386 return self._result
/usr/lib/python3.6/concurrent/futures/thread.py in run(self)
54
55 try:
---> 56 result = self.fn(*self.args, **self.kwargs)
57 except BaseException as exc:
58 self.future.set_exception(exc)
/usr/local/lib/python3.6/dist-packages/eolearn/core/eodata_io.py in <lambda>(loader)
74 if not lazy_loading:
75 with concurrent.futures.ThreadPoolExecutor() as executor:
---> 76 loading_data = executor.map(lambda loader: loader.load(), loading_data)
77
78 for (ftype, fname, _), value in zip(features, loading_data):
/usr/local/lib/python3.6/dist-packages/eolearn/core/eodata_io.py in load(self)
217 return self._decode(gzip_fp, self.path)
218
--> 219 return self._decode(file_handle, self.path)
220
221 def save(self, data, file_format, compress_level=0):
/usr/local/lib/python3.6/dist-packages/eolearn/core/eodata_io.py in _decode(file, path)
268
269 if FileFormat.NPY.extension() in path:
--> 270 return np.load(file)
271
272 raise ValueError('Unsupported data type.')
/usr/local/lib/python3.6/dist-packages/numpy/lib/npyio.py in load(file, mmap_mode, allow_pickle, fix_imports, encoding)
434 _ZIP_SUFFIX = b'PK\x05\x06' # empty zip files start with this
435 N = len(format.MAGIC_PREFIX)
--> 436 magic = fid.read(N)
437 # If the file size is less than N, we need to make sure not
438 # to seek past the beginning of the file
OSError: [Errno 5] Input/output error
Also some notebooks won't open and this appears instead:
I looked around at similar posts here, but didn't understand anything. Therefore, any help would be highly appreciated.
PS: my files are in subfolders and not directly contained in 'My Drive'. I have a lso disabled all the adblocks ad the problem persists...
I think the file/link has been used for downloading beyond its weekly limit. This answer may help you.
Some discussion about Google's policy on hosting data on drive.
The solution is to wait for couple of hours/days and try again.
I want to train a tensorflow image segmentation model on COCO, and thought I would leverage the dataset builder already included. Download seems to be completed but it crashes on extracting the zip files.
Running with TF 2.0.0 on a Jupyter Notebook under a conda environment. Computer is 64-bit Windows 10. The Oxford Pet III dataset used in the official image segmentation tutorial works fine.
Below is the error message (my local user name replaced with %user%).
---------------------------------------------------------------------------
OutOfRangeError Traceback (most recent call last)
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_datasets\core\download\extractor.py in _sync_extract(self, from_path, method, to_path)
88 try:
---> 89 for path, handle in iter_archive(from_path, method):
90 path = tf.compat.as_text(path)
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_datasets\core\download\extractor.py in iter_zip(arch_f)
176 with _open_or_pass(arch_f) as fobj:
--> 177 z = zipfile.ZipFile(fobj)
178 for member in z.infolist():
~\.conda\envs\tf-tutorial\lib\zipfile.py in __init__(self, file, mode, compression, allowZip64)
1130 if mode == 'r':
-> 1131 self._RealGetContents()
1132 elif mode in ('w', 'x'):
~\.conda\envs\tf-tutorial\lib\zipfile.py in _RealGetContents(self)
1193 try:
-> 1194 endrec = _EndRecData(fp)
1195 except OSError:
~\.conda\envs\tf-tutorial\lib\zipfile.py in _EndRecData(fpin)
263 # Determine file size
--> 264 fpin.seek(0, 2)
265 filesize = fpin.tell()
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_core\python\util\deprecation.py in new_func(*args, **kwargs)
506 instructions)
--> 507 return func(*args, **kwargs)
508
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_core\python\lib\io\file_io.py in seek(self, offset, whence, position)
166 elif whence == 2:
--> 167 offset += self.size()
168 else:
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_core\python\lib\io\file_io.py in size(self)
101 """Returns the size of the file."""
--> 102 return stat(self.__name).length
103
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_core\python\lib\io\file_io.py in stat(filename)
726 """
--> 727 return stat_v2(filename)
728
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_core\python\lib\io\file_io.py in stat_v2(path)
743 file_statistics = pywrap_tensorflow.FileStatistics()
--> 744 pywrap_tensorflow.Stat(compat.as_bytes(path), file_statistics)
745 return file_statistics
OutOfRangeError: C:\Users\%user%\tensorflow_datasets\downloads\images.cocodataset.org_zips_train20147eQIfmQL3bpVDgkOrnAQklNLVUtCsFrDPwMAuYSzF3U.zip; Unknown error
During handling of the above exception, another exception occurred:
ExtractError Traceback (most recent call last)
<ipython-input-27-887fa0198611> in <module>
1 cocoBuilder = tfds.builder('coco')
2 info = cocoBuilder.info
----> 3 cocoBuilder.download_and_prepare()
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_datasets\core\api_utils.py in disallow_positional_args_dec(fn, instance, args, kwargs)
50 _check_no_positional(fn, args, ismethod, allowed=allowed)
51 _check_required(fn, kwargs)
---> 52 return fn(*args, **kwargs)
53
54 return disallow_positional_args_dec(wrapped) # pylint: disable=no-value-for-parameter
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_datasets\core\dataset_builder.py in download_and_prepare(self, download_dir, download_config)
285 self._download_and_prepare(
286 dl_manager=dl_manager,
--> 287 download_config=download_config)
288
289 # NOTE: If modifying the lines below to put additional information in
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_datasets\core\dataset_builder.py in _download_and_prepare(self, dl_manager, download_config)
946 super(GeneratorBasedBuilder, self)._download_and_prepare(
947 dl_manager=dl_manager,
--> 948 max_examples_per_split=download_config.max_examples_per_split,
949 )
950
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_datasets\core\dataset_builder.py in _download_and_prepare(self, dl_manager, **prepare_split_kwargs)
802 # Generating data for all splits
803 split_dict = splits_lib.SplitDict()
--> 804 for split_generator in self._split_generators(dl_manager):
805 if splits_lib.Split.ALL == split_generator.split_info.name:
806 raise ValueError(
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_datasets\image\coco.py in _split_generators(self, dl_manager)
237 root_url = 'http://images.cocodataset.org/'
238 extracted_paths = dl_manager.download_and_extract({
--> 239 key: root_url + url for key, url in urls.items()
240 })
241
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_datasets\core\download\download_manager.py in download_and_extract(self, url_or_urls)
357 with self._downloader.tqdm():
358 with self._extractor.tqdm():
--> 359 return _map_promise(self._download_extract, url_or_urls)
360
361 #property
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_datasets\core\download\download_manager.py in _map_promise(map_fn, all_inputs)
393 """Map the function into each element and resolve the promise."""
394 all_promises = utils.map_nested(map_fn, all_inputs) # Apply the function
--> 395 res = utils.map_nested(_wait_on_promise, all_promises)
396 return res
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_datasets\core\utils\py_utils.py in map_nested(function, data_struct, dict_only, map_tuple)
127 return {
128 k: map_nested(function, v, dict_only, map_tuple)
--> 129 for k, v in data_struct.items()
130 }
131 elif not dict_only:
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_datasets\core\utils\py_utils.py in <dictcomp>(.0)
127 return {
128 k: map_nested(function, v, dict_only, map_tuple)
--> 129 for k, v in data_struct.items()
130 }
131 elif not dict_only:
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_datasets\core\utils\py_utils.py in map_nested(function, data_struct, dict_only, map_tuple)
141 return tuple(mapped)
142 # Singleton
--> 143 return function(data_struct)
144
145
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_datasets\core\download\download_manager.py in _wait_on_promise(p)
377
378 def _wait_on_promise(p):
--> 379 return p.get()
380
381 else:
~\.conda\envs\tf-tutorial\lib\site-packages\promise\promise.py in get(self, timeout)
508 target = self._target()
509 self._wait(timeout or DEFAULT_TIMEOUT)
--> 510 return self._target_settled_value(_raise=True)
511
512 def _target_settled_value(self, _raise=False):
~\.conda\envs\tf-tutorial\lib\site-packages\promise\promise.py in _target_settled_value(self, _raise)
512 def _target_settled_value(self, _raise=False):
513 # type: (bool) -> Any
--> 514 return self._target()._settled_value(_raise)
515
516 _value = _reason = _target_settled_value
~\.conda\envs\tf-tutorial\lib\site-packages\promise\promise.py in _settled_value(self, _raise)
222 if _raise:
223 raise_val = self._fulfillment_handler0
--> 224 reraise(type(raise_val), raise_val, self._traceback)
225 return self._fulfillment_handler0
226
~\.conda\envs\tf-tutorial\lib\site-packages\six.py in reraise(tp, value, tb)
694 if value.__traceback__ is not tb:
695 raise value.with_traceback(tb)
--> 696 raise value
697 finally:
698 value = None
~\.conda\envs\tf-tutorial\lib\site-packages\promise\promise.py in handle_future_result(future)
840 # type: (Any) -> None
841 try:
--> 842 resolve(future.result())
843 except Exception as e:
844 tb = exc_info()[2]
~\.conda\envs\tf-tutorial\lib\concurrent\futures\_base.py in result(self, timeout)
423 raise CancelledError()
424 elif self._state == FINISHED:
--> 425 return self.__get_result()
426
427 self._condition.wait(timeout)
~\.conda\envs\tf-tutorial\lib\concurrent\futures\_base.py in __get_result(self)
382 def __get_result(self):
383 if self._exception:
--> 384 raise self._exception
385 else:
386 return self._result
~\.conda\envs\tf-tutorial\lib\concurrent\futures\thread.py in run(self)
54
55 try:
---> 56 result = self.fn(*self.args, **self.kwargs)
57 except BaseException as exc:
58 self.future.set_exception(exc)
~\.conda\envs\tf-tutorial\lib\site-packages\tensorflow_datasets\core\download\extractor.py in _sync_extract(self, from_path, method, to_path)
92 except BaseException as err:
93 msg = 'Error while extracting %s to %s : %s' % (from_path, to_path, err)
---> 94 raise ExtractError(msg)
95 # `tf.io.gfile.Rename(overwrite=True)` doesn't work for non empty
96 # directories, so delete destination first, if it already exists.
ExtractError: Error while extracting C:\Users\%user%\tensorflow_datasets\downloads\images.cocodataset.org_zips_train20147eQIfmQL3bpVDgkOrnAQklNLVUtCsFrDPwMAuYSzF3U.zip to C:\Users\%user%\tensorflow_datasets\downloads\extracted\ZIP.images.cocodataset.org_zips_train20147eQIfmQL3bpVDgkOrnAQklNLVUtCsFrDPwMAuYSzF3U.zip : C:\Users\%user%\tensorflow_datasets\downloads\images.cocodataset.org_zips_train20147eQIfmQL3bpVDgkOrnAQklNLVUtCsFrDPwMAuYSzF3U.zip; Unknown error
The message seems cryptic to me. The folder to which it is trying to extract does not exist when the notebook is started - it is created by Tensorflow, and only at that command line. I obviously tried deleting it completely and running it again, to no effect.
The code that leads to the error is (everything runs fine until the last line):
import tensorflow as tf
from __future__ import absolute_import, division, print_function, unicode_literals
from tensorflow_examples.models.pix2pix import pix2pix
import tensorflow_datasets as tfds
from IPython.display import clear_output
import matplotlib.pyplot as plt
dataset, info = tfds.load('coco', with_info=True)
Also tried breaking down the last command into assigning the tdfs.builder object and then running download_and_extract, and again got the same error.
There is enough space in disk - after download, still 50+GB available, while the dataset is supposed to be 37GB in its largest version (2014).
I have a similar problem with Windows 10 & COCO 2017. My solution is simple. Extract the ZIP file manually according to the folder path in the error message.
I am a beginner developer who started to study automation by using pywinauto.
An overflow error occurs when using application.connect () to connect to an already open program.
But application.start() works fine....
Please help me if someone know this part.
The source code and error contents are as follows.
Source code:
import pywinauto
app = pywinauto.application.Application()
app.connect(title_re='Calculator')
Error:
OverflowError Traceback (most recent call last)
in
1 import pywinauto
2 app = pywinauto.application.Application()
----> 3 app.connect(title_re='Calculator')
d:\Anaconda3\lib\site-packages\pywinauto\application.py in connect(self, **kwargs)
972 ).process_id
973 else:
--> 974 self.process = findwindows.find_element(**kwargs).process_id
975 connected = True
976
d:\Anaconda3\lib\site-packages\pywinauto\findwindows.py in find_element(**kwargs)
82 so please see :py:func:find_elements for the full parameters description.
83 """
---> 84 elements = find_elements(**kwargs)
85
86 if not elements:
d:\Anaconda3\lib\site-packages\pywinauto\findwindows.py in find_elements(class_name, class_name_re, parent, process, title, title_re, top_level_only, visible_only, enabled_only, best_match, handle, ctrl_index, found_index, predicate_func, active_only, control_id, control_type, auto_id, framework_id, backend, depth)
279 return title_regex.match(t)
280 return False
--> 281 elements = [elem for elem in elements if _title_match(elem)]
282
283 if visible_only:
d:\Anaconda3\lib\site-packages\pywinauto\findwindows.py in (.0)
279 return title_regex.match(t)
280 return False
--> 281 elements = [elem for elem in elements if _title_match(elem)]
282
283 if visible_only:
d:\Anaconda3\lib\site-packages\pywinauto\findwindows.py in _title_match(w)
275 def _title_match(w):
276 """Match a window title to the regexp"""
--> 277 t = w.rich_text
278 if t is not None:
279 return title_regex.match(t)
d:\Anaconda3\lib\site-packages\pywinauto\win32_element_info.py in rich_text(self)
81 def rich_text(self):
82 """Return the text of the window"""
---> 83 return handleprops.text(self.handle)
84
85 name = rich_text
d:\Anaconda3\lib\site-packages\pywinauto\handleprops.py in text(handle)
86 length += 1
87
---> 88 buffer_ = ctypes.create_unicode_buffer(length)
89
90 ret = win32functions.SendMessage(
d:\Anaconda3\lib\ctypes_init_.py in create_unicode_buffer(init, size)
286 return buf
287 elif isinstance(init, int):
--> 288 buftype = c_wchar * init
289 buf = buftype()
290 return buf
OverflowError: cannot fit 'int' into an index-sized integer
import pywinauto
app = pywinauto.Application(backend='uia').start('calc.exe')
try this if you are having problem you have to say the backennd is a uia it working fine for me.
I've just been moving some code over to a Ubuntu 16.04.2 anaconda setup, and am getting a type error I don't understand when calling code which works fine across numerous other machines.
The error replicates for me just off of the list all tennis markets sample code in the repo below, as well as a request like:
from betfair import Betfair
client = Betfair("app key needed here", "path to ssh key here")
client.login(username, password)
client.keep_alive()
client.list_market_book(market_ids=['1.135391020'], price_projection=dict(priceData=['EX_BEST_OFFERS']))
or
from betfair.models import MarketFilter
event_types = client.list_event_types(
MarketFilter(text_query='tennis')
)
print(len(event_types)) # 2
print(event_types[0].event_type.name) # 'Tennis'
tennis_event_type = event_types[0]
markets = client.list_market_catalogue(
MarketFilter(event_type_ids=[tennis_event_type.event_type.id])
)
markets[0].market_name
Both throw the following type error despite identical code working on a windows installation:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-3-69b51cf78438> in <module>()
1 event_types = client.list_event_types(
----> 2 MarketFilter(text_query='tennis')
3 )
4 print(len(event_types)) # 2
5 print(event_types[0].event_type.name) # 'Tennis'
<decorator-gen-125> in list_event_types(self, filter, locale)
/home/user/anaconda2/lib/python2.7/site-packages/betfair/utils.pyc in requires_login(func, *args, **kwargs)
121 self = args[0]
122 if self.session_token:
--> 123 return func(*args, **kwargs)
124 raise exceptions.NotLoggedIn()
/home/user/anaconda2/lib/python2.7/site-packages/betfair/betfair.pyc in list_event_types(self, filter, locale)
148 'listEventTypes',
149 utils.get_kwargs(locals()),
--> 150 model=models.EventTypeResult,
151 )
152
/home/user/anaconda2/lib/python2.7/site-packages/betfair/betfair.pyc in make_api_request(self, base, method, params, codes, model)
87 utils.check_status_code(response, codes=codes)
88 result = utils.result_or_error(response)
---> 89 return utils.process_result(result, model)
90
91 # Authentication methods
/home/user/anaconda2/lib/python2.7/site-packages/betfair/utils.pyc in process_result(result, model)
81 return result
82 if isinstance(result, collections.Sequence):
---> 83 return [model(**item) for item in result]
84 return model(**result)
85
/home/user/anaconda2/lib/python2.7/site-packages/betfair/meta/models.pyc in __init__(self, **data)
24 def __init__(self, **data):
25 super(BetfairModel, self).__init__()
---> 26 self.import_data(data)
27
28 def import_data(self, data, **kwargs):
/home/user/anaconda2/lib/python2.7/site-packages/betfair/meta/models.pyc in import_data(self, data, **kwargs)
28 def import_data(self, data, **kwargs):
29 kwargs['strict'] = False
---> 30 return super(BetfairModel, self).import_data(data, **kwargs)
/home/user/anaconda2/lib/python2.7/site-packages/schematics/models.pyc in import_data(self, raw_data, recursive, **kwargs)
269 The data to be imported.
270 """
--> 271 data = self._convert(raw_data, trusted_data=_dict(self), recursive=recursive, **kwargs)
272 self._data.converted.update(data)
273 if kwargs.get('validate'):
/home/user/anaconda2/lib/python2.7/site-packages/schematics/models.pyc in _convert(self, raw_data, context, **kwargs)
293 should_validate = getattr(context, 'validate', kwargs.get('validate', False))
294 func = validate if should_validate else convert
--> 295 return func(self._schema, self, raw_data=raw_data, oo=True, context=context, **kwargs)
296
297 def export(self, field_converter=None, role=None, app_data=None, **kwargs):
/home/user/anaconda2/lib/python2.7/site-packages/schematics/transforms.pyc in convert(cls, mutable, raw_data, **kwargs)
427
428 def convert(cls, mutable, raw_data=None, **kwargs):
--> 429 return import_loop(cls, mutable, raw_data, import_converter, **kwargs)
430
431
/home/user/anaconda2/lib/python2.7/site-packages/schematics/transforms.pyc in import_loop(schema, mutable, raw_data, field_converter, trusted_data, mapping, partial, strict, init_values, apply_defaults, convert, validate, new, oo, recursive, app_data, context)
153 field_context = context
154 try:
--> 155 value = _field_converter(field, value, field_context)
156 except (FieldError, CompoundError) as exc:
157 errors[serialized_field_name] = exc
/home/user/anaconda2/lib/python2.7/site-packages/schematics/transforms.pyc in __call__(self, *args)
354
355 def __call__(self, *args):
--> 356 return self.func(*args)
357
358
/home/user/anaconda2/lib/python2.7/site-packages/schematics/transforms.pyc in import_converter(field, value, context)
382 if value is None or value is Undefined:
383 return value
--> 384 return field.convert(value, context)
385
386
/home/user/anaconda2/lib/python2.7/site-packages/schematics/types/compound.pyc in convert(self, value, context)
34 def convert(self, value, context=None):
35 context = context or get_import_context()
---> 36 return self._convert(value, context)
37
38 def _convert(self, value, context):
/home/user/anaconda2/lib/python2.7/site-packages/schematics/types/compound.pyc in _convert(self, value, context)
131 "Input must be a mapping or '%s' instance" % self.model_class.__name__)
132 if context.convert and context.oo:
--> 133 return model_class(value, context=context)
134 else:
135 return model_class.convert(value, context=context)
TypeError: __init__() takes exactly 1 argument (3 given)
Somewhat weirder, a request like:
client.list_market_catalogue(MarketFilter(market_ids=['1.135391020']))
Works fine.
python 2.7.13, Anaconda 4.4.0, Ubuntu 16.04.2
Any idea what could be causing this?
From the trace it looks to me like the schematics library is your issue. Checking the open issues on the betfair github there is, as of this writing, an open ticket regarding schematics breaking the api. It would appear that the author has left schematics out of the requirements and that version 1.1.1 is required. My guess is you have schematics 2.0 installed on the computer causing the issue.
One way to find this in the future would be to pip freeze the working environment and diff the broken environment. Moreover, when moving to a new machine you can use the output of pip freeze to duplicate the environment and avoid messy version issues like this.