issue_comments: 1469050607
This data as json
| html_url | issue_url | id | node_id | user | created_at | updated_at | author_association | body | reactions | performed_via_github_app | issue |
|---|---|---|---|---|---|---|---|---|---|---|---|
| https://github.com/pydata/xarray/pull/7019#issuecomment-1469050607 | https://api.github.com/repos/pydata/xarray/issues/7019 | 1469050607 | IC_kwDOAMm_X85Xj-7v | 35968931 | 2023-03-15T00:30:08Z | 2023-03-15T10:03:11Z | MEMBER | I tried opening a zarr store into xarray with chunking via cubed, but I got an error inside the indexing adapter classes. Somehow the type is completely wrong - would be good to type hint this part of the code, because this happens despite mypy passing now. ```python create example zarr storeorig = xr.tutorial.open_dataset("air_temperature") orig.to_zarr('air2.zarr') open it as a cubed arrayds = xr.open_dataset('air2.zarr', engine='zarr', chunks={}, from_array_kwargs={'manager': 'cubed'}) fails at this pointds.load() ```
```python
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
File ~/miniconda3/envs/cubed/lib/python3.9/site-packages/tenacity/__init__.py:382, in Retrying.__call__(self, fn, *args, **kwargs)
381 try:
--> 382 result = fn(*args, **kwargs)
383 except BaseException: # noqa: B902
File ~/miniconda3/envs/cubed/lib/python3.9/site-packages/cubed/runtime/executors/python.py:10, in exec_stage_func(func, *args, **kwargs)
8 @retry(stop=stop_after_attempt(3))
9 def exec_stage_func(func, *args, **kwargs):
---> 10 return func(*args, **kwargs)
File ~/miniconda3/envs/cubed/lib/python3.9/site-packages/cubed/primitive/blockwise.py:66, in apply_blockwise(out_key, config)
64 args.append(arg)
---> 66 result = config.function(*args)
67 if isinstance(result, dict): # structured array with named fields
File ~/miniconda3/envs/cubed/lib/python3.9/site-packages/cubed/core/ops.py:439, in map_blocks.<locals>.func_with_block_id.<locals>.wrap(*a, **kw)
438 block_id = offset_to_block_id(a[-1].item())
--> 439 return func(*a[:-1], block_id=block_id, **kw)
File ~/miniconda3/envs/cubed/lib/python3.9/site-packages/cubed/core/ops.py:572, in map_direct.<locals>.new_func.<locals>.wrap(block_id, *a, **kw)
571 args = a + arrays
--> 572 return func(*args, block_id=block_id, **kw)
File ~/miniconda3/envs/cubed/lib/python3.9/site-packages/cubed/core/ops.py:76, in _from_array(e, x, outchunks, asarray, block_id)
75 def _from_array(e, x, outchunks=None, asarray=None, block_id=None):
---> 76 out = x[get_item(outchunks, block_id)]
77 if asarray:
File ~/Documents/Work/Code/xarray/xarray/core/indexing.py:627, in CopyOnWriteArray.__getitem__(self, key)
626 def __getitem__(self, key):
--> 627 return type(self)(_wrap_numpy_scalars(self.array[key]))
File ~/Documents/Work/Code/xarray/xarray/core/indexing.py:534, in LazilyIndexedArray.__getitem__(self, indexer)
533 return array[indexer]
--> 534 return type(self)(self.array, self._updated_key(indexer))
File ~/Documents/Work/Code/xarray/xarray/core/indexing.py:500, in LazilyIndexedArray._updated_key(self, new_key)
499 def _updated_key(self, new_key):
--> 500 iter_new_key = iter(expanded_indexer(new_key.tuple, self.ndim))
501 full_key = []
AttributeError: 'tuple' object has no attribute 'tuple'
The above exception was the direct cause of the following exception:
RetryError Traceback (most recent call last)
Cell In[69], line 1
----> 1 ds.load()
File ~/Documents/Work/Code/xarray/xarray/core/dataset.py:761, in Dataset.load(self, **kwargs)
758 chunkmanager = get_chunked_array_type(*lazy_data.values())
760 # evaluate all the chunked arrays simultaneously
--> 761 evaluated_data = chunkmanager.compute(*lazy_data.values(), **kwargs)
763 for k, data in zip(lazy_data, evaluated_data):
764 self.variables[k].data = data
File ~/Documents/Work/Code/xarray/xarray/core/parallelcompat.py:451, in CubedManager.compute(self, *data, **kwargs)
448 def compute(self, *data: "CubedArray", **kwargs) -> np.ndarray:
449 from cubed import compute
--> 451 return compute(*data, **kwargs)
File ~/miniconda3/envs/cubed/lib/python3.9/site-packages/cubed/core/array.py:300, in compute(executor, callbacks, optimize_graph, *arrays, **kwargs)
297 executor = PythonDagExecutor()
299 _return_in_memory_array = kwargs.pop("_return_in_memory_array", True)
--> 300 plan.execute(
301 executor=executor,
302 callbacks=callbacks,
303 optimize_graph=optimize_graph,
304 array_names=[a.name for a in arrays],
305 **kwargs,
306 )
308 if _return_in_memory_array:
309 return tuple(a._read_stored() for a in arrays)
File ~/miniconda3/envs/cubed/lib/python3.9/site-packages/cubed/core/plan.py:154, in Plan.execute(self, executor, callbacks, optimize_graph, array_names, **kwargs)
152 if callbacks is not None:
153 [callback.on_compute_start(dag) for callback in callbacks]
--> 154 executor.execute_dag(
155 dag, callbacks=callbacks, array_names=array_names, **kwargs
156 )
157 if callbacks is not None:
158 [callback.on_compute_end(dag) for callback in callbacks]
File ~/miniconda3/envs/cubed/lib/python3.9/site-packages/cubed/runtime/executors/python.py:22, in PythonDagExecutor.execute_dag(self, dag, callbacks, array_names, **kwargs)
20 if stage.mappable is not None:
21 for m in stage.mappable:
---> 22 exec_stage_func(stage.function, m, config=pipeline.config)
23 if callbacks is not None:
24 event = TaskEndEvent(array_name=name)
File ~/miniconda3/envs/cubed/lib/python3.9/site-packages/tenacity/__init__.py:289, in BaseRetrying.wraps.<locals>.wrapped_f(*args, **kw)
287 @functools.wraps(f)
288 def wrapped_f(*args: t.Any, **kw: t.Any) -> t.Any:
--> 289 return self(f, *args, **kw)
File ~/miniconda3/envs/cubed/lib/python3.9/site-packages/tenacity/__init__.py:379, in Retrying.__call__(self, fn, *args, **kwargs)
377 retry_state = RetryCallState(retry_object=self, fn=fn, args=args, kwargs=kwargs)
378 while True:
--> 379 do = self.iter(retry_state=retry_state)
380 if isinstance(do, DoAttempt):
381 try:
File ~/miniconda3/envs/cubed/lib/python3.9/site-packages/tenacity/__init__.py:326, in BaseRetrying.iter(self, retry_state)
324 if self.reraise:
325 raise retry_exc.reraise()
--> 326 raise retry_exc from fut.exception()
328 if self.wait:
329 sleep = self.wait(retry_state)
RetryError: RetryError[<Future at 0x7fc0c69be4f0 state=finished raised AttributeError>
```
This still works fine for dask. |
{
"total_count": 0,
"+1": 0,
"-1": 0,
"laugh": 0,
"hooray": 0,
"confused": 0,
"heart": 0,
"rocket": 0,
"eyes": 0
} |
1368740629 |