Skip to content

icap_mme

ICAP-MME Reader

ICAPMMEReader

Bases: GriddedReader

Reader for ICAP-MME (International Cooperative for Aerosol Prediction - Multi Model Ensemble) data.

Source code in monetio/readers/icap_mme.py 
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
@register_reader("icap_mme")
class ICAPMMEReader(GriddedReader):
    """
    Reader for ICAP-MME (International Cooperative for Aerosol Prediction - Multi Model Ensemble) data.
    """

    def open_dataset(
        self,
        files: str | list[str] | None = None,
        dates: pd.DatetimeIndex | list[datetime] | datetime | str | None = None,
        product: str = "MMC",
        data_var: str = "dustaod550",
        download: bool = False,
        **kwargs: Any,
    ) -> xr.Dataset:
        """
        Retrieve and load ICAP-MME data.

        Parameters
        ----------
        files : Union[str, List[str]], optional
            File paths or URLs to read. If None, uses `dates` and `product` to discover files.
        dates : Union[pd.DatetimeIndex, List[datetime], datetime, str], optional
            Dates to retrieve if `files` is not provided.
        product : str, optional
            ICAP product (e.g., 'MMC', 'C4', 'MME'), by default 'MMC'.
        data_var : str, optional
            Data variable (e.g., 'dustaod550'), by default 'dustaod550'.
        download : bool, optional
            Whether to download files to local directory, by default False.
        **kwargs : Any
            Additional arguments passed to the driver.

        Returns
        -------
        xr.Dataset
            The loaded ICAP-MME dataset.

        Examples
        --------
        >>> reader = ICAPMMEReader()
        >>> ds = reader.open_dataset(dates="2024-02-01", product="C4")
        """
        if files is None:
            if dates is None:
                raise ValueError("Either 'files' or 'dates' must be provided.")

            # Validate parameters before building URLs
            if product.upper() not in VALID_FILETYPES:
                raise ValueError(
                    f"Invalid input for 'product': '{product}'. Must be one of {VALID_FILETYPES}."
                )
            if data_var.lower() not in VALID_DATA_VARS:
                raise ValueError(
                    f"Invalid input for 'data_var': '{data_var}'. Must be one of {VALID_DATA_VARS}."
                )

            urls, fnames = build_urls(dates, filetype=product, data_var=data_var)
            if download:
                files = []
                for url, fname in zip(urls, fnames):
                    files.append(str(retrieve(url, fname, download=True)))
            else:
                files = urls

        # ICAP files are standard NetCDF, often h5netcdf compatible.
        if "concat_dim" not in kwargs:
            kwargs["concat_dim"] = "time"
        if "combine" not in kwargs:
            kwargs["combine"] = "nested"

        # Use XarrayDriver to open (Lazy by default)
        ds = self.driver.open(files, **kwargs)

        ds = self.harmonize(ds)

        # Update history
        ds = update_history(ds, "Read ICAP-MME data.")

        return ds

open_dataset(files=None, dates=None, product='MMC', data_var='dustaod550', download=False, **kwargs)

Retrieve and load ICAP-MME data.

Parameters:

Name Type Description Default
files Union[str, List[str]]

File paths or URLs to read. If None, uses dates and product to discover files.

 None
dates Union[DatetimeIndex, List[datetime], datetime, str]

Dates to retrieve if files is not provided.

 None
product str

ICAP product (e.g., 'MMC', 'C4', 'MME'), by default 'MMC'.

 'MMC'
data_var str

Data variable (e.g., 'dustaod550'), by default 'dustaod550'.

 'dustaod550'
download bool

Whether to download files to local directory, by default False.

 False
**kwargs Any

Additional arguments passed to the driver.

 {}

Returns:

Type Description
Dataset

The loaded ICAP-MME dataset.

Examples:

>>> reader = ICAPMMEReader()
>>> ds = reader.open_dataset(dates="2024-02-01", product="C4")
Source code in monetio/readers/icap_mme.py 
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
def open_dataset(
    self,
    files: str | list[str] | None = None,
    dates: pd.DatetimeIndex | list[datetime] | datetime | str | None = None,
    product: str = "MMC",
    data_var: str = "dustaod550",
    download: bool = False,
    **kwargs: Any,
) -> xr.Dataset:
    """
    Retrieve and load ICAP-MME data.

    Parameters
    ----------
    files : Union[str, List[str]], optional
        File paths or URLs to read. If None, uses `dates` and `product` to discover files.
    dates : Union[pd.DatetimeIndex, List[datetime], datetime, str], optional
        Dates to retrieve if `files` is not provided.
    product : str, optional
        ICAP product (e.g., 'MMC', 'C4', 'MME'), by default 'MMC'.
    data_var : str, optional
        Data variable (e.g., 'dustaod550'), by default 'dustaod550'.
    download : bool, optional
        Whether to download files to local directory, by default False.
    **kwargs : Any
        Additional arguments passed to the driver.

    Returns
    -------
    xr.Dataset
        The loaded ICAP-MME dataset.

    Examples
    --------
    >>> reader = ICAPMMEReader()
    >>> ds = reader.open_dataset(dates="2024-02-01", product="C4")
    """
    if files is None:
        if dates is None:
            raise ValueError("Either 'files' or 'dates' must be provided.")

        # Validate parameters before building URLs
        if product.upper() not in VALID_FILETYPES:
            raise ValueError(
                f"Invalid input for 'product': '{product}'. Must be one of {VALID_FILETYPES}."
            )
        if data_var.lower() not in VALID_DATA_VARS:
            raise ValueError(
                f"Invalid input for 'data_var': '{data_var}'. Must be one of {VALID_DATA_VARS}."
            )

        urls, fnames = build_urls(dates, filetype=product, data_var=data_var)
        if download:
            files = []
            for url, fname in zip(urls, fnames):
                files.append(str(retrieve(url, fname, download=True)))
        else:
            files = urls

    # ICAP files are standard NetCDF, often h5netcdf compatible.
    if "concat_dim" not in kwargs:
        kwargs["concat_dim"] = "time"
    if "combine" not in kwargs:
        kwargs["combine"] = "nested"

    # Use XarrayDriver to open (Lazy by default)
    ds = self.driver.open(files, **kwargs)

    ds = self.harmonize(ds)

    # Update history
    ds = update_history(ds, "Read ICAP-MME data.")

    return ds

build_urls(dates, filetype='MMC', data_var='dustaod550', verbose=True)

Construct ICAP-MME URLs and filenames for the given dates.

Parameters:

Name Type Description Default
dates Union[DatetimeIndex, List[datetime], datetime, str]

Dates to build URLs for.

 required
filetype str

ICAP product type (MMC, C4, MME), by default "MMC".

 'MMC'
data_var str

Data variable name, by default "dustaod550".

 'dustaod550'
verbose bool

Whether to print status messages, by default True.

 True

Returns:

Type Description
Tuple[List[str], List[str]]

A tuple containing a list of URLs and a list of filenames.

Examples:

>>> urls, fnames = build_urls("2024-02-01", filetype="C4")
Source code in monetio/readers/icap_mme.py 
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
def build_urls(
    dates: pd.DatetimeIndex | list[datetime] | datetime | str,
    filetype: str = "MMC",
    data_var: str = "dustaod550",
    verbose: bool = True,
) -> tuple[list[str], list[str]]:
    """
    Construct ICAP-MME URLs and filenames for the given dates.

    Parameters
    ----------
    dates : Union[pd.DatetimeIndex, List[datetime], datetime, str]
        Dates to build URLs for.
    filetype : str, optional
        ICAP product type (MMC, C4, MME), by default "MMC".
    data_var : str, optional
        Data variable name, by default "dustaod550".
    verbose : bool, optional
        Whether to print status messages, by default True.

    Returns
    -------
    Tuple[List[str], List[str]]
        A tuple containing a list of URLs and a list of filenames.

    Examples
    --------
    >>> urls, fnames = build_urls("2024-02-01", filetype="C4")
    """
    from collections.abc import Iterable

    if isinstance(dates, Iterable) and not isinstance(dates, str):
        dates = pd.DatetimeIndex(dates)
    else:
        dates = pd.DatetimeIndex([dates])

    urls = []
    fnames = []
    if verbose:
        print("Building ICAP-MME URLs...")
    base_url = "https://usgodae.org/ftp/outgoing/nrl/ICAP-MME/"

    for dt in dates:
        fname = "icap_{}_{}_{}.nc".format(
            dt.strftime(r"%Y%m%d%H"), filetype.upper(), data_var.lower()
        )
        url = base_url + dt.strftime(r"%Y/%Y%m/") + fname
        urls.append(url)
        fnames.append(fname)

    return urls, fnames

retrieve(url, fname, download=False, verbose=True)

Retrieve or download an ICAP-MME file.

Parameters:

Name Type Description Default
url str

Source URL.

 required
fname str

Target filename.

 required
download bool

Whether to download to a local file, by default False.

 False
verbose bool

Whether to print status, by default True.

 True

Returns:

Type Description
Union[str, Path]

The path or URL to the file.

Examples:

>>> path = retrieve("https://.../file.nc", "file.nc", download=True)
Source code in monetio/readers/icap_mme.py 
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
def retrieve(url: str, fname: str, download: bool = False, verbose: bool = True) -> str | Path:
    """
    Retrieve or download an ICAP-MME file.

    Parameters
    ----------
    url : str
        Source URL.
    fname : str
        Target filename.
    download : bool, optional
        Whether to download to a local file, by default False.
    verbose : bool, optional
        Whether to print status, by default True.

    Returns
    -------
    Union[str, Path]
        The path or URL to the file.

    Examples
    --------
    >>> path = retrieve("https://.../file.nc", "file.nc", download=True)
    """
    p = Path(fname).absolute()
    fs = FileUtility.get_fs(url)

    if not download:
        # Return URL for remote opening via fsspec
        return url
    else:
        if not p.is_file():
            if verbose:
                print(f"Downloading {url} to {p.as_posix()}")
            fs.get(url, str(p))
        else:
            if verbose:
                print(f"File Exists: {p.as_posix()}")
        return p