issue_comments

Well the error occurs while writing the file. So if the reason is, that the file is corrupt then necdf4 is corrupting it... right? The problem is rather, that it does not even finish writing the file.

FYI: The error does not occur at the same file every time.

Have a good weekend!! ;)

Hi there,

I just wanted to let you know, that I do get the same error, when working with the following script. It does not use xarray or rasterio. So those will most likely not be the problem.

```python class Gif2NetCDF(): def init(self, netcdf_name, netcdf_destfolder, gif_folder, gif_filter_pattern=None, detailled_conversion=True): """

    Parameters
    ----------
    netcdf_name : str
        specifying the name of the netcdf file to write.

    netcdf_destfolder : str
        specifying the folder where the netcdf file is supposed to be created

    giffolder : str
        specifying the folder that contains the gif files, that are supposed to be written as an netcdf

    gif_filter_pattern : str
        specifying a re pattern to filter the all files in directory so you end up with the gif files you want.
        The here specified string will be passed to re.match which will be checked for each file in the provided giffolder


    Examples
    --------

    ## defining variables
    import netCDF4
    netcdf_name = "2016-07.nc"
    netcdf_destfolder = "./example/radar"
    giffolder = "./example/radar/2016-07/"

    ## create Instance
    cdf = Gif2NetCDF(netcdf_name, netcdf_destfolder, giffolder,gif_filter_pattern)
    ## write all (filtered) gifs in folder to netcdf file
    cdf.writeCDF()
    """

    ## creating global vars

    self.netcdf_name = netcdf_name
    self.netcdf_destfolder = netcdf_destfolder
    self.giffolder = gif_folder
    self.refilterpattern = gif_filter_pattern
    self.detailled_conversion = detailled_conversion

    self.netcdfFP = os.path.join(netcdf_destfolder, netcdf_name)

    # preparing the coordinate vectors
    self._lat_range = np.arange(479500, -160500, step=-1000)  # north
    self._long_range = np.arange(255500, 965500, step=1000)  # east

    # preparing time origin
    self.time_origin = dt.datetime.strptime("1970-01-01 00:00:00", "%Y-%m-%d %H:%M:%S")

    self.raincodes = np.array([....]) # this array is quite large, so I left it out... Which unfortunately means the code does not run... If anybody needs it please let me know

    return

def list_gifs(self):
    self._gifs = os.listdir(self.giffolder)
    return self._gifs

def filter_gifs(self):
    self._gifs = [file for file in self._gifs if re.match(self.refilterpattern, file)]
    return self._gifs

def addDimensions(self):

    # adds dimensions to empty netcdf file
    self.latD = self.netcdfFile.createDimension('lat', self._lat_range.shape[0])  # north-south
    self.lonD = self.netcdfFile.createDimension('lon', self._long_range.shape[0])  # east-west
    self.timeD = self.netcdfFile.createDimension('time', None)

    return

def addVariables(self):

    ## creating variables
    self.latV = self.netcdfFile.createVariable("chy", np.float32, ("lat",), complevel=9, zlib=True)  # north-south
    self.lonV = self.netcdfFile.createVariable("chx", np.float32, ("lon",), complevel=9, zlib=True)  # east-west
    self.timeV = self.netcdfFile.createVariable("time", np.float64, ("time",), complevel=9, zlib=True)

    self.rainV = self.netcdfFile.createVariable("rain", np.float32, ("time", "lat", "lon"), complevel=9, zlib=True,
                                                fill_value=-100)

    ## adding units
    self.latV.units = "meters"
    self.lonV.units = "meters"

    self.timeV.units = "seconds"
    self.timeV.calender = "standard"

    self.rainV.units = "millimeter/hour"

    ## adding longname
    self.latV.long_name = "swiss northing CH1903"
    self.lonV.long_name = "swiss easting CH1903"
    self.timeV.long_name = "seconds since 1970-01-01 00:00:00"

    self.rainV.long_name = "precipitation intensity forecast"

    return

def addDescription(self):

    self.netcdfFile.description = """..."""

    self.netcdfFile.history = """Created: {}""".format(dt.datetime.now().strftime("%Y-%m-%d %H:%M"))
    self.netcdfFile.source = '...'
    return

def _write_static_dims(self):
    self.latV[:] = self._lat_range
    self.lonV[:] = self._long_range
    return

def _write_time(self, file, datetime=None):

    if datetime is None:
        datestr = re.findall("\.([0-9]+)\.gif", file)[0]
        date = dt.datetime.strptime(datestr, "%Y%m%d%H%M")
    else:
        date = datetime

    seconds = (date - self.time_origin).total_seconds()
    current_size = self.timeV.size
    self.timeV[current_size] = seconds

    return

def gif2array(self, file):

    xpix = 0
    ypix = 76
    n_pixel_x = 710 + xpix
    n_pixel_y = 640 + ypix

    gif = np.array(Image.open(file))[ypix:n_pixel_y, xpix:n_pixel_x].astype("float64")
    for idx, raincode in enumerate(self.raincodes):
        gif[gif == idx] = raincode[3]
    return gif

def _write_rain(self, file):
    array = self.gif2array(os.path.join(self.giffolder, file))
    idx = self.rainV.shape[0] - 1
    self.rainV[idx, :, :] = array
    return

def writeCDF(self):
    self.netcdfFile = Dataset(self.netcdfFP, 'w', format='NETCDF4_CLASSIC', )
    try:
        giflist = self.list_gifs()
        if self.refilterpattern is not None:
            fgiflist = self.filter_gifs()

        self.addDimensions()
        self.addVariables()
        self.addDescription()
        self._write_static_dims()

        for file in tqdm(self._gifs):
            self._write_time(file)
            self._write_rain(file)

    except Exception:
        self.netcdfFile.close()
        raise

    self.netcdfFile.close()

    return

```

Error

python Traceback (most recent call last): . . . File "C:\Users\foerstch\AppData\Local\Programs\Python\Python37\lib\site-packages\archiving\radar.py", line 358, in _write_rain idx = self.rainV.shape[0] - 1 File "netCDF4\_netCDF4.pyx", line 4031, in netCDF4._netCDF4.Variable.shape.__get__ File "netCDF4\_netCDF4.pyx", line 3369, in netCDF4._netCDF4.Dimension.__len__ File "netCDF4\_netCDF4.pyx", line 1857, in netCDF4._netCDF4._ensure_nc_success RuntimeError: NetCDF: HDF error

Does anybody have some advice on how to fix this?

Thanks a bunch! Christian