html_url,issue_url,id,node_id,user,created_at,updated_at,author_association,body,reactions,performed_via_github_app,issue
https://github.com/pydata/xarray/issues/2535#issuecomment-499919008,https://api.github.com/repos/pydata/xarray/issues/2535,499919008,MDEyOklzc3VlQ29tbWVudDQ5OTkxOTAwOA==,9096348,2019-06-07T15:00:08Z,2019-06-07T15:00:08Z,NONE,"Well the error occurs while writing the file. So if the reason is, that the file is corrupt then necdf4 is corrupting it... right? The problem is rather, that it does not even finish writing the file.
FYI: The error does not occur at the same file every time.
Have a good weekend!! ;)","{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,376389539
https://github.com/pydata/xarray/issues/2535#issuecomment-499876672,https://api.github.com/repos/pydata/xarray/issues/2535,499876672,MDEyOklzc3VlQ29tbWVudDQ5OTg3NjY3Mg==,9096348,2019-06-07T13:04:33Z,2019-06-07T13:04:33Z,NONE,"Hi there,
I just wanted to let you know, that I do get the same error, when working with the following script. It does not use xarray or rasterio. So those will most likely not be the problem.
```python
class Gif2NetCDF():
def __init__(self, netcdf_name, netcdf_destfolder, gif_folder,
gif_filter_pattern=None, detailled_conversion=True):
""""""
Parameters
----------
netcdf_name : str
specifying the name of the netcdf file to write.
netcdf_destfolder : str
specifying the folder where the netcdf file is supposed to be created
giffolder : str
specifying the folder that contains the gif files, that are supposed to be written as an netcdf
gif_filter_pattern : str
specifying a re pattern to filter the all files in directory so you end up with the gif files you want.
The here specified string will be passed to re.match which will be checked for each file in the provided giffolder
Examples
--------
## defining variables
import netCDF4
netcdf_name = ""2016-07.nc""
netcdf_destfolder = ""./example/radar""
giffolder = ""./example/radar/2016-07/""
## create Instance
cdf = Gif2NetCDF(netcdf_name, netcdf_destfolder, giffolder,gif_filter_pattern)
## write all (filtered) gifs in folder to netcdf file
cdf.writeCDF()
""""""
## creating global vars
self.netcdf_name = netcdf_name
self.netcdf_destfolder = netcdf_destfolder
self.giffolder = gif_folder
self.refilterpattern = gif_filter_pattern
self.detailled_conversion = detailled_conversion
self.netcdfFP = os.path.join(netcdf_destfolder, netcdf_name)
# preparing the coordinate vectors
self._lat_range = np.arange(479500, -160500, step=-1000) # north
self._long_range = np.arange(255500, 965500, step=1000) # east
# preparing time origin
self.time_origin = dt.datetime.strptime(""1970-01-01 00:00:00"", ""%Y-%m-%d %H:%M:%S"")
self.raincodes = np.array([....]) # this array is quite large, so I left it out... Which unfortunately means the code does not run... If anybody needs it please let me know
return
def list_gifs(self):
self._gifs = os.listdir(self.giffolder)
return self._gifs
def filter_gifs(self):
self._gifs = [file for file in self._gifs if re.match(self.refilterpattern, file)]
return self._gifs
def addDimensions(self):
# adds dimensions to empty netcdf file
self.latD = self.netcdfFile.createDimension('lat', self._lat_range.shape[0]) # north-south
self.lonD = self.netcdfFile.createDimension('lon', self._long_range.shape[0]) # east-west
self.timeD = self.netcdfFile.createDimension('time', None)
return
def addVariables(self):
## creating variables
self.latV = self.netcdfFile.createVariable(""chy"", np.float32, (""lat"",), complevel=9, zlib=True) # north-south
self.lonV = self.netcdfFile.createVariable(""chx"", np.float32, (""lon"",), complevel=9, zlib=True) # east-west
self.timeV = self.netcdfFile.createVariable(""time"", np.float64, (""time"",), complevel=9, zlib=True)
self.rainV = self.netcdfFile.createVariable(""rain"", np.float32, (""time"", ""lat"", ""lon""), complevel=9, zlib=True,
fill_value=-100)
## adding units
self.latV.units = ""meters""
self.lonV.units = ""meters""
self.timeV.units = ""seconds""
self.timeV.calender = ""standard""
self.rainV.units = ""millimeter/hour""
## adding longname
self.latV.long_name = ""swiss northing CH1903""
self.lonV.long_name = ""swiss easting CH1903""
self.timeV.long_name = ""seconds since 1970-01-01 00:00:00""
self.rainV.long_name = ""precipitation intensity forecast""
return
def addDescription(self):
self.netcdfFile.description = """"""...""""""
self.netcdfFile.history = """"""Created: {}"""""".format(dt.datetime.now().strftime(""%Y-%m-%d %H:%M""))
self.netcdfFile.source = '...'
return
def _write_static_dims(self):
self.latV[:] = self._lat_range
self.lonV[:] = self._long_range
return
def _write_time(self, file, datetime=None):
if datetime is None:
datestr = re.findall(""\.([0-9]+)\.gif"", file)[0]
date = dt.datetime.strptime(datestr, ""%Y%m%d%H%M"")
else:
date = datetime
seconds = (date - self.time_origin).total_seconds()
current_size = self.timeV.size
self.timeV[current_size] = seconds
return
def gif2array(self, file):
xpix = 0
ypix = 76
n_pixel_x = 710 + xpix
n_pixel_y = 640 + ypix
gif = np.array(Image.open(file))[ypix:n_pixel_y, xpix:n_pixel_x].astype(""float64"")
for idx, raincode in enumerate(self.raincodes):
gif[gif == idx] = raincode[3]
return gif
def _write_rain(self, file):
array = self.gif2array(os.path.join(self.giffolder, file))
idx = self.rainV.shape[0] - 1
self.rainV[idx, :, :] = array
return
def writeCDF(self):
self.netcdfFile = Dataset(self.netcdfFP, 'w', format='NETCDF4_CLASSIC', )
try:
giflist = self.list_gifs()
if self.refilterpattern is not None:
fgiflist = self.filter_gifs()
self.addDimensions()
self.addVariables()
self.addDescription()
self._write_static_dims()
for file in tqdm(self._gifs):
self._write_time(file)
self._write_rain(file)
except Exception:
self.netcdfFile.close()
raise
self.netcdfFile.close()
return
```
**Error**
```python
Traceback (most recent call last):
.
.
.
File ""C:\Users\foerstch\AppData\Local\Programs\Python\Python37\lib\site-packages\archiving\radar.py"", line 358, in _write_rain
idx = self.rainV.shape[0] - 1
File ""netCDF4\_netCDF4.pyx"", line 4031, in netCDF4._netCDF4.Variable.shape.__get__
File ""netCDF4\_netCDF4.pyx"", line 3369, in netCDF4._netCDF4.Dimension.__len__
File ""netCDF4\_netCDF4.pyx"", line 1857, in netCDF4._netCDF4._ensure_nc_success
RuntimeError: NetCDF: HDF error
```
Does anybody have some advice on how to fix this?
Thanks a bunch!
Christian","{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,376389539