geoms

GEOMS Reader

GEOMSReader

Bases: GriddedReader

Reader for GEOMS format files (HDF4/HDF5).

Source code in monetio/readers/geoms.py

@register_reader("geoms")
class GEOMSReader(GriddedReader):
    """
    Reader for GEOMS format files (HDF4/HDF5).
    """

    def open_dataset(
        self,
        files: str | list[str],
        rename_all: bool = True,
        squeeze: bool = True,
        **kwargs: Any,
    ) -> xr.Dataset:
        """
        Reads GEOMS format files.

        Parameters
        ----------
        files : Union[str, List[str]]
            File path, list of paths, or glob pattern.
        rename_all : bool, optional
            Whether to rename all variables to lowercase/standard names, by default True.
        squeeze : bool, optional
            Whether to squeeze dimensions of size 1, by default True.
        **kwargs : Any
            Additional arguments passed to the driver.

        Returns
        -------
        xr.Dataset
            The processed GEOMS dataset.

        Examples
        --------
        >>> reader = GEOMSReader()
        >>> ds = reader.open_dataset("groundbased_uvvis.doas.directsun.no2*.h5")
        """
        # Note: GEOMS files (especially HDF4) often require custom logic that
        # open_mfdataset might not handle natively without a complex engine.
        # We maintain a loop but ensure laziness where possible.

        file_list = FileUtility.expand_paths(files)

        dsets = []
        for f in file_list:
            ds = open_dataset_geoms(f, rename_all=rename_all, squeeze=squeeze)
            dsets.append(ds)

        if not dsets:
            return xr.Dataset()

        if len(dsets) == 1:
            ds = dsets[0]
        else:
            ds = xr.concat(dsets, dim="time")

        # Update history
        ds = update_history(ds, "Read GEOMS data.")

        return ds

open_dataset(files, rename_all=True, squeeze=True, **kwargs)

Reads GEOMS format files.

Parameters:

Name	Type	Description	Default
files	Union[str, List[str]]	File path, list of paths, or glob pattern.	required
rename_all	bool	Whether to rename all variables to lowercase/standard names, by default True.	True
squeeze	bool	Whether to squeeze dimensions of size 1, by default True.	True
**kwargs	Any	Additional arguments passed to the driver.	{}

Returns:

Type	Description
Dataset	The processed GEOMS dataset.

Examples:

>>> reader = GEOMSReader()
>>> ds = reader.open_dataset("groundbased_uvvis.doas.directsun.no2*.h5")

Source code in monetio/readers/geoms.py

def open_dataset(
    self,
    files: str | list[str],
    rename_all: bool = True,
    squeeze: bool = True,
    **kwargs: Any,
) -> xr.Dataset:
    """
    Reads GEOMS format files.

    Parameters
    ----------
    files : Union[str, List[str]]
        File path, list of paths, or glob pattern.
    rename_all : bool, optional
        Whether to rename all variables to lowercase/standard names, by default True.
    squeeze : bool, optional
        Whether to squeeze dimensions of size 1, by default True.
    **kwargs : Any
        Additional arguments passed to the driver.

    Returns
    -------
    xr.Dataset
        The processed GEOMS dataset.

    Examples
    --------
    >>> reader = GEOMSReader()
    >>> ds = reader.open_dataset("groundbased_uvvis.doas.directsun.no2*.h5")
    """
    # Note: GEOMS files (especially HDF4) often require custom logic that
    # open_mfdataset might not handle natively without a complex engine.
    # We maintain a loop but ensure laziness where possible.

    file_list = FileUtility.expand_paths(files)

    dsets = []
    for f in file_list:
        ds = open_dataset_geoms(f, rename_all=rename_all, squeeze=squeeze)
        dsets.append(ds)

    if not dsets:
        return xr.Dataset()

    if len(dsets) == 1:
        ds = dsets[0]
    else:
        ds = xr.concat(dsets, dim="time")

    # Update history
    ds = update_history(ds, "Read GEOMS data.")

    return ds

geoms_preprocess(ds, *, rename_all=True, squeeze=True)

Standardizes GEOMS dataset coordinates and dimensions.

Parameters:

Name	Type	Description	Default
ds	Dataset	Input GEOMS dataset.	required
rename_all	bool	Whether to rename all variables, by default True.	True
squeeze	bool	Whether to squeeze, by default True.	True

Returns:

Type	Description
Dataset	Preprocessed dataset.

Source code in monetio/readers/geoms.py

def geoms_preprocess(
    ds: xr.Dataset, *, rename_all: bool = True, squeeze: bool = True
) -> xr.Dataset:
    """
    Standardizes GEOMS dataset coordinates and dimensions.

    Parameters
    ----------
    ds : xr.Dataset
        Input GEOMS dataset.
    rename_all : bool, optional
        Whether to rename all variables, by default True.
    squeeze : bool, optional
        Whether to squeeze, by default True.

    Returns
    -------
    xr.Dataset
        Preprocessed dataset.
    """
    # 1. Handle Instrument Coordinates
    instru_coords = [
        "LATITUDE.INSTRUMENT",
        "LONGITUDE.INSTRUMENT",
        "ALTITUDE.INSTRUMENT",
    ]
    for vn in instru_coords:
        if vn in ds:
            da = ds[vn]
            if da.ndim == 0:
                ds = ds.set_coords(vn)
                continue
            (dim_name0,) = da.dims
            dim_name = _rename_var(vn)
            ds = ds.set_coords(vn).rename_dims({dim_name0: dim_name})

    # 2. Main Dimension Renaming
    rename_main_dims = {"DATETIME": "time", "ALTITUDE": "altitude"}
    actual_dim_renames = {}
    for ref, new_dim in rename_main_dims.items():
        if ref not in ds:
            continue
        n = ds[ref].size
        time_dims = [
            dim_name
            for dim_name, dim_size in ds.sizes.items()
            if dim_name.startswith("fakeDim") and dim_size == n
        ]
        for td in time_dims:
            actual_dim_renames[td] = new_dim

    if actual_dim_renames:
        ds = ds.rename_dims(actual_dim_renames)
        ds = update_history(ds, f"Renamed dimensions: {actual_dim_renames}")

    # 3. Squeeze singleton dimensions if they look like placeholders
    for vn, da in ds.variables.items():
        if da.ndim >= 1 and da.dims[-1].startswith("fakeDim") and da.dtype.kind == "f":
            n = da.sizes[da.dims[-1]]
            if n == 1:
                ds[vn] = da.squeeze(dim=da.dims[-1])

    # 4. Handle String Variables (Backend-agnostic)
    ds = _handle_strings(ds)

    # 5. Type and Byte Order Cleanup
    for vn, da in ds.variables.items():
        if da.dtype.kind == "f":
            if da.dtype.byteorder not in {"=", "|"}:
                ds[vn] = da.astype(da.dtype.newbyteorder("="))

    # 6. Time Conversion (Lazy)
    ds = _convert_times_lazy(ds)
    ds = update_history(ds, "Converted MJD2000 times to datetime64[ns] lazily.")

    # 7. Final Renaming and Squeezing
    rename_vars = {k: v for k, v in rename_main_dims.items() if k in ds.variables}
    rename_vars.update({old: _rename_var(old) for old in instru_coords if old in ds})
    if rename_vars:
        ds = ds.rename_vars(rename_vars)

    if rename_all:
        ds = ds.rename_vars({old: _rename_var(old) for old in ds.data_vars})

    if "latitude_instrument" in ds.coords and "latitude" not in ds.coords:
        ds = ds.rename(
            {
                "latitude_instrument": "latitude",
                "longitude_instrument": "longitude",
            }
        )

    if squeeze:
        ds = ds.squeeze()

    # Update history
    ds = update_history(ds, "Preprocessed GEOMS data.")

    return ds

open_dataset_geoms(fp, *, rename_all=True, squeeze=True)

Internal function to open a single GEOMS file.

Parameters:

Name	Type	Description	Default
fp	str	File path.	required
rename_all	bool	Whether to rename all variables, by default True.	True
squeeze	bool	Whether to squeeze, by default True.	True

Returns:

Type	Description
Dataset	Processed dataset.

Source code in monetio/readers/geoms.py

def open_dataset_geoms(fp: str, *, rename_all: bool = True, squeeze: bool = True) -> xr.Dataset:
    """
    Internal function to open a single GEOMS file.

    Parameters
    ----------
    fp : str
        File path.
    rename_all : bool, optional
        Whether to rename all variables, by default True.
    squeeze : bool, optional
        Whether to squeeze, by default True.

    Returns
    -------
    xr.Dataset
        Processed dataset.
    """
    from monetio.util import _import_required

    ext = Path(fp).suffix.lower()
    fs = FileUtility.get_fs(fp)

    if ext in {".h4", ".hdf4", ".hdf"}:
        pyhdf_SD = _import_required("pyhdf.SD")
        if fp.startswith("s3://"):
            import tempfile

            with tempfile.NamedTemporaryFile(suffix=ext, delete=True) as tmp:
                fs.get(fp, tmp.name)
                sd = pyhdf_SD.SD(tmp.name)
                data_vars, attrs = _read_hdf4(sd)
                sd.end()
        else:
            sd = pyhdf_SD.SD(str(fp))
            data_vars, attrs = _read_hdf4(sd)
            sd.end()

        # For HDF4, we have NumPy arrays now. Wrap in Dataset.
        ds = xr.Dataset(data_vars=data_vars, attrs=attrs)

    elif ext in {".h5", ".he5", ".hdf5"}:
        # For HDF5, we try to use xarray's native lazy loading if possible,
        # but GEOMS HDF5 structure (using dimension labels) often needs manual intervention.
        from monetio.util import _import_required

        h5py = _import_required("h5py")
        da_array = _import_required("dask.array")

        f_obj = fs.open(fp, "rb")
        f = h5py.File(f_obj, "r")

        data_vars = {}
        for k, v in f.items():
            dims = tuple(_rename_h5_dim(str(d)) for d in v.dims)
            # We wrap in DataArray with lazy loading using dask.array.from_array.
            # This allows backend-agnostic lazy evaluation without immediate v[...] compute.
            data_lazy = da_array.from_array(v, chunks="auto")
            data_vars[k] = (dims, data_lazy, dict(v.attrs))

        attrs = dict(f.attrs)
        # Note: We don't close the file yet because dask needs the handle for lazy reads.
        # This is a trade-off for GEOMS HDF5 files which don't fit standard xr.open_dataset engines.
        ds = xr.Dataset(data_vars=data_vars, attrs=attrs)
    else:
        raise ValueError(f"unrecognized file extension: {ext!r}")

    ds = geoms_preprocess(ds, rename_all=rename_all, squeeze=squeeze)

    return ds