issues: 254732241
This data as json
id | node_id | number | title | user | state | locked | assignee | milestone | comments | created_at | updated_at | closed_at | author_association | active_lock_reason | draft | pull_request | body | reactions | performed_via_github_app | state_reason | repo | type |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
254732241 | MDU6SXNzdWUyNTQ3MzIyNDE= | 1547 | ValueError when saving multiindexed datasets in netcdf | 14169057 | closed | 0 | 1 | 2017-09-01T19:32:11Z | 2017-11-13T03:13:36Z | 2017-11-13T03:13:36Z | NONE | Given this test code below (it reconstructs how I receive my datasets so ignore the complicated part at the beginning, just look at the variable ``` import xarray as xr import numpy as np import pandas as pd import seaborn as sns # pandas aware plotting library from copy import deepcopy import time np.random.seed(123) def strTimeProp(start, end, format, prop): """Get a time at a proportion of a range of two formatted times.
def randomDate(start, end): return strTimeProp(start, end, '%d/%m/%Y %I:%M %p', np.random.random()) n_stars = 100 stars = list(range(n_stars)) pointings_targets = {0: [0], 1: [1], 2: [2, 3]} pointing_exposures = {0: {'iband': 30, 'ha1':34}, 1: {'iband': 29, 'ha1':31}, 2: {'iband': 24, 'ha2': 33}} detection_efficiency = 0.95 columns = list('abcdefghi') pointings = list(pointings_targets.keys()) targets = [j for i in pointings_targets.values() for j in i] n_pointings = len(pointings) n_targets = len(targets) what stars are in each pointingpointing_stars = {} for pointing in pointings: pointing_stars[pointing] = pointings_targets[pointing] # contains the targets stars_to_choose_from = list(set(stars) - set(pointing_stars[pointing])) # other stars to choose chosen_stars = np.random.choice(stars_to_choose_from, np.random.randint(5, len(stars_to_choose_from)), replace=False).tolist() pointing_stars[pointing] += chosen_stars # list(set(chosen_stars) | set(pointing_stars[pointing])) # stars in a pointing data = [] indices = [] for pointing, bands in pointing_exposures.items(): for band_name, n_exps in bands.items(): dates = sorted([randomDate('1/1/2015 1:00 AM', '30/1/2015 1:00 AM') for _ in range(n_exps)]) for i, date in enumerate(dates): stars_detected = np.random.choice(pointing_stars[pointing], int(detection_efficiency*len(pointing_stars[pointing])), replace=False) values = np.random.randn(len(stars_detected), len(columns)) table = pd.DataFrame(values, columns=columns, index=stars_detected) # as if from file table.index.name = 'objid' arr = table.to_xarray().to_array() indices.append((pointing, band_name, date)) data.append(arr) concat = xr.concat(data, pd.MultiIndex.from_tuples(indices, names=['pointing', 'band', 'date'])).to_dataset('variable') o = concat.coords['objid'].values
other = pd.DataFrame(np.random.normal(0, 1, size=(len(o), 2)), index=o, columns=['ra', 'dec'])
other.index.name = 'objid'
concat = concat.merge(other.to_xarray()) # dataframe
concat.to_netcdf('test')
C:\Anaconda3\envs\test_xarray\lib\site-packages\xarray\core\dataset.py in to_netcdf(self, path, mode, format, group, engine, encoding, unlimited_dims) 975 return to_netcdf(self, path, mode, format=format, group=group, 976 engine=engine, encoding=encoding, --> 977 unlimited_dims=unlimited_dims) 978 979 def unicode(self): C:\Anaconda3\envs\test_xarray\lib\site-packages\xarray\backends\api.py in to_netcdf(dataset, path_or_file, mode, format, group, engine, writer, encoding, unlimited_dims) 571 try: 572 dataset.dump_to_store(store, sync=sync, encoding=encoding, --> 573 unlimited_dims=unlimited_dims) 574 if path_or_file is None: 575 return target.getvalue() C:\Anaconda3\envs\test_xarray\lib\site-packages\xarray\core\dataset.py in dump_to_store(self, store, encoder, sync, encoding, unlimited_dims) 914 915 store.store(variables, attrs, check_encoding, --> 916 unlimited_dims=unlimited_dims) 917 if sync: 918 store.sync() C:\Anaconda3\envs\test_xarray\lib\site-packages\xarray\backends\common.py in store(self, variables, attributes, args, kwargs) 242 # All NetCDF files get CF encoded by default, without this attempting 243 # to write times, for example, would fail. --> 244 cf_variables, cf_attrs = cf_encoder(variables, attributes) 245 AbstractWritableDataStore.store(self, cf_variables, cf_attrs, 246 args, **kwargs) C:\Anaconda3\envs\test_xarray\lib\site-packages\xarray\conventions.py in cf_encoder(variables, attributes) 1087 """ 1088 new_vars = OrderedDict((k, encode_cf_variable(v, name=k)) -> 1089 for k, v in iteritems(variables)) 1090 return new_vars, attributes C:\Anaconda3\envs\test_xarray\lib\site-packages\xarray\conventions.py in <genexpr>(.0) 1087 """ 1088 new_vars = OrderedDict((k, encode_cf_variable(v, name=k)) -> 1089 for k, v in iteritems(variables)) 1090 return new_vars, attributes C:\Anaconda3\envs\test_xarray\lib\site-packages\xarray\conventions.py in encode_cf_variable(var, needs_copy, name) 739 var = maybe_default_fill_value(var) 740 var = maybe_encode_bools(var) --> 741 var = ensure_dtype_not_object(var) 742 return var 743 C:\Anaconda3\envs\test_xarray\lib\site-packages\xarray\conventions.py in ensure_dtype_not_object(var) 707 data[missing] = fill_value 708 else: --> 709 data = data.astype(dtype=_infer_dtype(data)) 710 var = Variable(dims, data, attrs, encoding) 711 return var ValueError: setting an array element with a sequence ``` |
{ "url": "https://api.github.com/repos/pydata/xarray/issues/1547/reactions", "total_count": 0, "+1": 0, "-1": 0, "laugh": 0, "hooray": 0, "confused": 0, "heart": 0, "rocket": 0, "eyes": 0 } |
completed | 13221727 | issue |