issues: 303809308

This data as json

id node_id number title user state locked assignee milestone comments created_at updated_at closed_at author_association active_lock_reason draft pull_request body reactions performed_via_github_app state_reason repo type
303809308 MDU6SXNzdWUzMDM4MDkzMDg= 1977 Netcdf char array not being decoded to string in compound dtype 4849151 open 0     3 2018-03-09T11:23:04Z 2020-02-14T13:26:11Z   NONE      

Code Sample, a copy-pastable example if possible

python script_nc_file = "bolo_geom_fromscript.nc" with xr.open_dataset(script_nc_file, group='/bolo/sxd', concat_characters=True) as ds: ds = ds.copy(deep=True) print(ds.slits) <xarray.DataArray 'slits' (slit_no: 4)> array([ ([b'B', b'o', b'l', b'o', b'm', b'e', b't', b'e', b'r', b'S', b'l', b'i', b't', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b''], [b'M', b'A', b'S', b'T', b'-', b'U', b' ', b'S', b'X', b'D', b' ', b'-', b' ', b'O', b'u', b't', b'e', b'r', b' ', b'S', b'l', b'i', b't', b' ', b'1', b'', b'', b'', b'', b''], 1, (-0.06458486, 0.21803484, -0.97380162), ( 0.95881973, 0.28401534, 0.), (-0.52069675, 1.77104629, -1.564 ), 0.005, 0.005, [b'M', b'A', b'S', b'T', b'-', b'U', b' ', b'S', b'X', b'D', b' ', b'-', b' ', b'O', b'u', b't', b'e', b'r', b' ', b'S', b'l', b'i', b't', b' ', b'1', b'', b'', b'', b'', b''], 0), ([b'B', b'o', b'l', b'o', b'm', b'e', b't', b'e', b'r', b'S', b'l', b'i', b't', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b''], [b'M', b'A', b'S', b'T', b'-', b'U', b' ', b'S', b'X', b'D', b' ', b'-', b' ', b'O', b'u', b't', b'e', b'r', b' ', b'S', b'l', b'i', b't', b' ', b'2', b'', b'', b'', b'', b''], 1, (-0.16038567, 0.54145294, -0.82529095), ( 0.95881973, 0.28401534, 0.), (-0.5278879 , 1.76891617, -1.564 ), 0.005, 0.005, [b'M', b'A', b'S', b'T', b'-', b'U', b' ', b'S', b'X', b'D', b' ', b'-', b' ', b'O', b'u', b't', b'e', b'r', b' ', b'S', b'l', b'i', b't', b' ', b'2', b'', b'', b'', b'', b''], 1), ([b'B', b'o', b'l', b'o', b'm', b'e', b't', b'e', b'r', b'S', b'l', b'i', b't', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b''], [b'M', b'A', b'S', b'T', b'-', b'U', b' ', b'S', b'X', b'D', b' ', b'-', b' ', b'U', b'p', b'p', b'e', b'r', b' ', b'S', b'l', b'i', b't', b' ', b'3', b'', b'', b'', b'', b''], 1, (-0.26470454, 0.89362754, -0.36243804), ( 0.95881973, 0.28401534, 0.), (-0.31231469, 1.06756025, -1.57072314), 0.005, 0.005, [b'M', b'A', b'S', b'T', b'-', b'U', b' ', b'S', b'X', b'D', b' ', b'-', b' ', b'U', b'p', b'p', b'e', b'r', b' ', b'S', b'l', b'i', b't', b' ', b'3', b'', b'', b'', b'', b''], 2), ([b'B', b'o', b'l', b'o', b'm', b'e', b't', b'e', b'r', b'S', b'l', b'i', b't', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b'', b''], [b'M', b'A', b'S', b'T', b'-', b'U', b' ', b'S', b'X', b'D', b' ', b'-', b' ', b'U', b'p', b'p', b'e', b'r', b' ', b'S', b'l', b'i', b't', b' ', b'4', b'', b'', b'', b'', b''], 1, (-0.19640032, 0.66303636, 0.72236396), ( 0.95881973, 0.28401534, 0.), (-0.31950584, 1.06543013, -1.57072314), 0.005, 0.005, [b'M', b'A', b'S', b'T', b'-', b'U', b' ', b'S', b'X', b'D', b' ', b'-', b' ', b'U', b'p', b'p', b'e', b'r', b' ', b'S', b'l', b'i', b't', b' ', b'4', b'', b'', b'', b'', b''], 3)], dtype={'names':['Object_type','ID','Version','basis_1','basis_2','centre_point','width','height','slit_id','slit_no'], 'formats':[('S1', (30,)),('S1', (30,)),'<i4',[('x', '<f8'), ('y', '<f8'), ('z', '<f8')],[('x', '<f8'), ('y', '<f8'), ('z', '<f8')],[('x', '<f8'), ('y', '<f8'), ('z', '<f8')],'<f8','<f8',('S1', (30,)),'<i4'], 'offsets':[0,30,60,64,88,112,136,144,152,184], 'itemsize':192, 'aligned':True}) Coordinates: * slit_no (slit_no) int64 0 1 2 3 Attributes: units: m

Problem description

When opening the attached dataset ,the char arrays in the compound dtype are not being converted into strings, despite concat_characters=True. bolo_geom_fromscript.nc.zip

Expected Output

The char arrays should be converted into strings (or at the very least, bytes if an encoding is not present): ```python import netCDF4 as nc longest_str = np.max([ds.slits.values[field].shape[-1] for field in ds.slits.values.dtype.fields if ds.slits.values[field].dtype.kind in ('S', 'U')]) str_dtype = '<U{}'.format(longest_str) cartesian_coord = np.dtype([('x', np.float64), ('y', np.float64), ('z', np.float64)]) aperture_dtype_str = np.dtype([('Object_type', str_dtype), ('ID', str_dtype), ('Version', np.int32), ('basis_1', cartesian_coord), ('basis_2', cartesian_coord), ('centre_point', cartesian_coord), ('width', np.float64), ('height', np.float64), ('slit_id', str_dtype), ('slit_no', np.int32)]) ds['slits_str'] = xr.DataArray(np.empty(ds.slits.size, aperture_dtype_str), coords=[('slit_no', ds.coords['slit_no'])]) for key in ds.slits.values.dtype.fields: if key in ('Object_type', 'ID', 'slit_id'): string_key = nc.chartostring(ds.slits.values[key]) ds.slits_str.values[key] = string_key else: ds.slits_str.values[key] = ds.slits.values[key] print(ds.slits_str)

<xarray.DataArray 'slits_str' (slit_no: 4)> array([ ('BolometerSlit', 'MAST-U SXD - Outer Slit 1', 1, (-0.06458486, 0.21803484, -0.97380162), ( 0.95881973, 0.28401534, 0.), (-0.52069675, 1.77104629, -1.564 ), 0.005, 0.005, 'MAST-U SXD - Outer Slit 1', 0), ('BolometerSlit', 'MAST-U SXD - Outer Slit 2', 1, (-0.16038567, 0.54145294, -0.82529095), ( 0.95881973, 0.28401534, 0.), (-0.5278879 , 1.76891617, -1.564 ), 0.005, 0.005, 'MAST-U SXD - Outer Slit 2', 1), ('BolometerSlit', 'MAST-U SXD - Upper Slit 3', 1, (-0.26470454, 0.89362754, -0.36243804), ( 0.95881973, 0.28401534, 0.), (-0.31231469, 1.06756025, -1.57072314), 0.005, 0.005, 'MAST-U SXD - Upper Slit 3', 2), ('BolometerSlit', 'MAST-U SXD - Upper Slit 4', 1, (-0.19640032, 0.66303636, 0.72236396), ( 0.95881973, 0.28401534, 0.), (-0.31950584, 1.06543013, -1.57072314), 0.005, 0.005, 'MAST-U SXD - Upper Slit 4', 3)], dtype=[('Object_type', '<U30'), ('ID', '<U30'), ('Version', '<i4'), ('basis_1', [('x', '<f8'), ('y', '<f8'), ('z', '<f8')]), ('basis_2', [('x', '<f8'), ('y', '<f8'), ('z', '<f8')]), ('centre_point', [('x', '<f8'), ('y', '<f8'), ('z', '<f8')]), ('width', '<f8'), ('height', '<f8'), ('slit_id', '<U30'), ('slit_no', '<i4')]) Coordinates: * slit_no (slit_no) int64 0 1 2 3 ```

Output of xr.show_versions()

INSTALLED VERSIONS ------------------ commit: None python: 3.6.4.final.0 python-bits: 64 OS: Linux OS-release: 4.13.0-32-generic machine: x86_64 processor: x86_64 byteorder: little LC_ALL: None LANG: en_GB.UTF-8 LOCALE: en_GB.UTF-8 xarray: 0.10.0 pandas: 0.22.0 numpy: 1.13.3 scipy: 1.0.0 netCDF4: 1.3.1 h5netcdf: None Nio: None bottleneck: 1.2.1 cyordereddict: None dask: 0.15.3 matplotlib: 2.1.0 cartopy: None seaborn: 0.8.0 setuptools: 38.5.1 pip: 9.0.1 conda: 4.4.11 pytest: 3.2.1 IPython: 6.2.1 sphinx: 1.6.3 
{
    "url": "https://api.github.com/repos/pydata/xarray/issues/1977/reactions",
    "total_count": 0,
    "+1": 0,
    "-1": 0,
    "laugh": 0,
    "hooray": 0,
    "confused": 0,
    "heart": 0,
    "rocket": 0,
    "eyes": 0
}
     13221727 issue

Links from other tables