Skip to content

umbc_aerosol

UMBC Aerosol Reader (CL51)

UMBCAerosolReader

Bases: GriddedReader

Reader for UMBC Aerosol (CL51 Ceilometer) data.

Source code in monetio/readers/umbc_aerosol.py 
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
@register_reader("umbc_aerosol")
class UMBCAerosolReader(GriddedReader):
    """
    Reader for UMBC Aerosol (CL51 Ceilometer) data.
    """

    def open_dataset(
        self,
        files: str | list[str],
        **kwargs,
    ) -> xr.Dataset:
        """
        Reads UMBC Aerosol HDF5 files.

        Parameters
        ----------
        files : Union[str, List[str]]
            File path(s) or URL(s).
        **kwargs : dict
            Additional arguments passed to XarrayDriver.open.

        Returns
        -------
        xr.Dataset
            The UMBC Aerosol dataset.

        Examples
        --------
        >>> reader = UMBCAerosolReader()
        >>> ds = reader.open_dataset(files="UMBC_CL51_*.h5")
        """
        # UMBC CL51 HDF5 files have 'DATA' and 'Instrument_Attributes' groups
        groups = ["DATA", "Instrument_Attributes"]

        user_preprocess = kwargs.pop("preprocess", None)

        if "engine" not in kwargs:
            kwargs["engine"] = "h5netcdf"

        dsets = []
        all_attrs = {}
        for g in groups:
            g_kwargs = kwargs.copy()
            g_kwargs["group"] = g
            try:
                # We open without the UMBC preprocess at this stage
                ds_g = super().open_dataset(files, **g_kwargs)
                dsets.append(ds_g)
                # Manually collect attributes from groups
                all_attrs.update(ds_g.attrs)
            except Exception as e:
                warnings.warn(f"Could not open group {g}: {e}")

        if not dsets:
            raise RuntimeError("No groups could be opened.")

        # Merge groups
        # We use compat='no_conflicts' as coordinates should be identical
        ds = xr.merge(dsets, compat="no_conflicts")
        ds.attrs.update(all_attrs)

        # Now apply UMBC preprocessing to the merged dataset
        ds = umbc_aerosol_preprocess(ds)

        if user_preprocess:
            ds = user_preprocess(ds)

        # Update history
        ds = update_history(ds, "Read UMBC Aerosol data.")

        return ds

open_dataset(files, **kwargs)

Reads UMBC Aerosol HDF5 files.

Parameters:

Name Type Description Default
files Union[str, List[str]]

File path(s) or URL(s).

 required
**kwargs dict

Additional arguments passed to XarrayDriver.open.

 {}

Returns:

Type Description
Dataset

The UMBC Aerosol dataset.

Examples:

>>> reader = UMBCAerosolReader()
>>> ds = reader.open_dataset(files="UMBC_CL51_*.h5")
Source code in monetio/readers/umbc_aerosol.py 
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
def open_dataset(
    self,
    files: str | list[str],
    **kwargs,
) -> xr.Dataset:
    """
    Reads UMBC Aerosol HDF5 files.

    Parameters
    ----------
    files : Union[str, List[str]]
        File path(s) or URL(s).
    **kwargs : dict
        Additional arguments passed to XarrayDriver.open.

    Returns
    -------
    xr.Dataset
        The UMBC Aerosol dataset.

    Examples
    --------
    >>> reader = UMBCAerosolReader()
    >>> ds = reader.open_dataset(files="UMBC_CL51_*.h5")
    """
    # UMBC CL51 HDF5 files have 'DATA' and 'Instrument_Attributes' groups
    groups = ["DATA", "Instrument_Attributes"]

    user_preprocess = kwargs.pop("preprocess", None)

    if "engine" not in kwargs:
        kwargs["engine"] = "h5netcdf"

    dsets = []
    all_attrs = {}
    for g in groups:
        g_kwargs = kwargs.copy()
        g_kwargs["group"] = g
        try:
            # We open without the UMBC preprocess at this stage
            ds_g = super().open_dataset(files, **g_kwargs)
            dsets.append(ds_g)
            # Manually collect attributes from groups
            all_attrs.update(ds_g.attrs)
        except Exception as e:
            warnings.warn(f"Could not open group {g}: {e}")

    if not dsets:
        raise RuntimeError("No groups could be opened.")

    # Merge groups
    # We use compat='no_conflicts' as coordinates should be identical
    ds = xr.merge(dsets, compat="no_conflicts")
    ds.attrs.update(all_attrs)

    # Now apply UMBC preprocessing to the merged dataset
    ds = umbc_aerosol_preprocess(ds)

    if user_preprocess:
        ds = user_preprocess(ds)

    # Update history
    ds = update_history(ds, "Read UMBC Aerosol data.")

    return ds

umbc_aerosol_preprocess(ds)

Preprocess UMBC Aerosol dataset: standardize coordinates and handle time.

Parameters:

Name Type Description Default
ds Dataset

Input dataset from merged 'DATA' and 'Instrument_Attributes' groups.

 required

Returns:

Type Description
Dataset

Processed dataset.
Source code in monetio/readers/umbc_aerosol.py 
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
def umbc_aerosol_preprocess(ds: xr.Dataset) -> xr.Dataset:
    """
    Preprocess UMBC Aerosol dataset: standardize coordinates and handle time.

    Parameters
    ----------
    ds : xr.Dataset
        Input dataset from merged 'DATA' and 'Instrument_Attributes' groups.

    Returns
    -------
    xr.Dataset
        Processed dataset.
    """
    # 1. Handle renaming and dimensions
    if "Altitude_m" in ds.variables:
        ds = ds.rename({"Altitude_m": "altitude"})
        if ds["altitude"].ndim == 1:
            a_dim = ds["altitude"].dims[0]
            if a_dim != "z":
                ds = ds.rename({a_dim: "z"})
            ds = ds.set_coords("altitude")

    if "Profile_bsc" in ds.variables:
        ds = ds.rename({"Profile_bsc": "bsc"})

    if "UnixTime_UTC" in ds.variables:

        def _convert_time(t):
            return pd.to_datetime(t, unit="s").astype("datetime64[ns]")

        time_da = apply_lazy_conversion(ds["UnixTime_UTC"], _convert_time, "datetime64[ns]")
        u_dim = ds["UnixTime_UTC"].dims[0]

        # Assign time as a coordinate and swap dimension
        ds = ds.assign_coords(time=time_da)
        if u_dim != "time":
            ds = ds.swap_dims({u_dim: "time"})

        if "UnixTime_UTC" in ds.data_vars:
            ds = ds.drop_vars("UnixTime_UTC")

    # 2. Handle Coordinates (Latitude/Longitude) from Attributes
    # Extract from merged attributes (from Instrument_Attributes group)
    lat = ds.attrs.get("Location_lat", 0.0)
    lon = ds.attrs.get("Location_lon", 0.0)

    # Handle case where attributes are stored as lists/arrays
    if isinstance(lat, list | np.ndarray) and len(lat) > 0:
        lat = lat[0]
    if isinstance(lon, list | np.ndarray) and len(lon) > 0:
        lon = lon[0]

    try:
        lat = float(lat)
        lon = float(lon)
    except (TypeError, ValueError):
        lat, lon = 0.0, 0.0

    # Ensure x and y dimensions exist for consistency with other lidar readers
    if "x" not in ds.dims:
        ds = ds.expand_dims("x")
        ds["x"] = [0.0]
    if "y" not in ds.dims:
        ds = ds.expand_dims("y")
        ds["y"] = [0.0]

    ds.coords["latitude"] = (("y", "x"), np.array([[lat]]))
    ds.coords["longitude"] = (("y", "x"), np.array([[lon]]))

    ds.coords["latitude"].attrs.update({"units": "degrees_north", "standard_name": "latitude"})
    ds.coords["longitude"].attrs.update({"units": "degrees_east", "standard_name": "longitude"})

    # Update history
    ds = update_history(ds, "Preprocessed UMBC Aerosol data using standardized preprocessing.")

    return ds