Skip to content

iagos

IAGOS Reader.

IAGOSReader

Bases: PointReader

IAGOS Data Reader following standard conventions. Supports both local files and retrieval via IAGOS API (placeholder).

Source code in monetio/readers/iagos.py 
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
@register_reader("iagos")
class IAGOSReader(PointReader):
    """
    IAGOS Data Reader following standard conventions.
    Supports both local files and retrieval via IAGOS API (placeholder).
    """

    fixed_location = False

    def open_dataset(
        self,
        files: str | list[str] | None = None,
        dates: pd.DatetimeIndex | list[datetime.datetime] | datetime.datetime | str | None = None,
        as_xarray: bool = True,
        lazy: bool = False,
        **kwargs,
    ) -> xr.Dataset | pd.DataFrame | dd.DataFrame:
        """
        Retrieve and load IAGOS data.

        Parameters
        ----------
        files : Union[str, List[str]], optional
            File path, list of paths, or glob pattern.
        dates : Union[pd.DatetimeIndex, List[datetime], datetime, str], optional
            Dates to retrieve if files are not provided.
        as_xarray : bool, optional
            Whether to return an xarray.Dataset, by default True.
        lazy : bool, optional
            Whether to return a dask-backed object, by default False.
        **kwargs : dict
            Additional arguments passed to the reader and driver.

        Returns
        -------
        Union[pd.DataFrame, xr.Dataset, dd.DataFrame]
            The loaded IAGOS data.
        """
        if files is None:
            if dates is None:
                raise ValueError("Must provide either 'files' or 'dates'.")
            files = self.build_urls(dates, **kwargs)

        if not files:
            if as_xarray:
                return xr.Dataset()
            return pd.DataFrame()

        # IAGOS data is NetCDF. We use xr.open_mfdataset for robustness.
        # It handles both single and multiple files, and laziness.
        ds = xr.open_mfdataset(files, combine="nested", concat_dim="time", **kwargs)

        ds = self.harmonize(ds)

        if not as_xarray:
            return ds.to_dataframe().reset_index()

        ds = update_history(ds, "Read IAGOS data using standardized preprocessing.")
        return ds

    def build_urls(
        self,
        dates: pd.DatetimeIndex | list[datetime.datetime] | datetime.datetime | str,
        **kwargs,
    ) -> list[str]:
        """
        Construct IAGOS URLs.
        Note: IAGOS usually requires registration. This implementation
        documents the required credentials for API access.

        Parameters
        ----------
        dates : Union[pd.DatetimeIndex, List[datetime], datetime, str]
            Dates to build URLs for.
        api_key : str, optional
            IAGOS API key. Can also be set via IAGOS_API_KEY environment variable.

        Returns
        -------
        List[str]
            List of matching IAGOS URLs.
        """
        api_key = kwargs.get("api_key") or os.environ.get("IAGOS_API_KEY")
        if not api_key:
            # We cannot build URLs without an API key or a public mirror.
            # For now, return empty and warn the user.
            import warnings

            warnings.warn(
                "IAGOS retrieval requires an API key. Please provide 'api_key' or set IAGOS_API_KEY env var."
            )
            return []

        # Implementation of IAGOS API retrieval would go here.
        # Example: https://services.iagos-data.fr/prod/v2.0/download?api_key=...
        return []

    def harmonize(self, ds: xr.Dataset) -> xr.Dataset:
        """
        Standardize IAGOS variable names and units.

        Parameters
        ----------
        ds : xr.Dataset
            Input dataset.

        Returns
        -------
        xr.Dataset
            Harmonized dataset.
        """
        # Common IAGOS to MONETIO mappings
        rename_dict = {
            "lon": "longitude",
            "lat": "latitude",
            "baro_alt": "altitude",
            "air_temp": "temperature",
            "o3": "ozone",
            "co": "carbon_monoxide",
            "h2o": "water_vapor",
            "no": "nitrogen_monoxide",
            "nox": "nitrogen_oxides",
            "gps_lat": "latitude",
            "gps_lon": "longitude",
            "gps_alt": "altitude",
        }

        # Filter rename_dict to only include variables present in ds
        actual_rename = {}
        for k, v in rename_dict.items():
            if k in ds.variables and v not in ds.variables:
                actual_rename[k] = v

        if actual_rename:
            ds = ds.rename(actual_rename)

        # Ensure coordinates are set correctly for PointReader expectations
        for coord in ["time", "latitude", "longitude", "altitude"]:
            if coord in ds.variables and coord not in ds.coords:
                ds = ds.set_coords(coord)

        # Standard units and metadata
        if "ozone" in ds.variables:
            # Check if units are already set, only override if sure
            if ds["ozone"].attrs.get("units") in ["ppb", "ppbv", "1e-9"]:
                ds["ozone"].attrs["units"] = "ppb"
                ds["ozone"].attrs["standard_name"] = "mole_fraction_of_ozone_in_air"

        if "carbon_monoxide" in ds.variables:
            if ds["carbon_monoxide"].attrs.get("units") in ["ppb", "ppbv", "1e-9"]:
                ds["carbon_monoxide"].attrs["units"] = "ppb"
                ds["carbon_monoxide"].attrs["standard_name"] = (
                    "mole_fraction_of_carbon_monoxide_in_air"
                )

        return ds

build_urls(dates, **kwargs)

Construct IAGOS URLs. Note: IAGOS usually requires registration. This implementation documents the required credentials for API access.

Parameters:

Name Type Description Default
dates Union[DatetimeIndex, List[datetime], datetime, str]

Dates to build URLs for.

 required
api_key str

IAGOS API key. Can also be set via IAGOS_API_KEY environment variable.

 required

Returns:

Type Description
List[str]

List of matching IAGOS URLs.
Source code in monetio/readers/iagos.py 
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
def build_urls(
    self,
    dates: pd.DatetimeIndex | list[datetime.datetime] | datetime.datetime | str,
    **kwargs,
) -> list[str]:
    """
    Construct IAGOS URLs.
    Note: IAGOS usually requires registration. This implementation
    documents the required credentials for API access.

    Parameters
    ----------
    dates : Union[pd.DatetimeIndex, List[datetime], datetime, str]
        Dates to build URLs for.
    api_key : str, optional
        IAGOS API key. Can also be set via IAGOS_API_KEY environment variable.

    Returns
    -------
    List[str]
        List of matching IAGOS URLs.
    """
    api_key = kwargs.get("api_key") or os.environ.get("IAGOS_API_KEY")
    if not api_key:
        # We cannot build URLs without an API key or a public mirror.
        # For now, return empty and warn the user.
        import warnings

        warnings.warn(
            "IAGOS retrieval requires an API key. Please provide 'api_key' or set IAGOS_API_KEY env var."
        )
        return []

    # Implementation of IAGOS API retrieval would go here.
    # Example: https://services.iagos-data.fr/prod/v2.0/download?api_key=...
    return []

harmonize(ds)

Standardize IAGOS variable names and units.

Parameters:

Name Type Description Default
ds Dataset

Input dataset.

 required

Returns:

Type Description
Dataset

Harmonized dataset.
Source code in monetio/readers/iagos.py 
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
def harmonize(self, ds: xr.Dataset) -> xr.Dataset:
    """
    Standardize IAGOS variable names and units.

    Parameters
    ----------
    ds : xr.Dataset
        Input dataset.

    Returns
    -------
    xr.Dataset
        Harmonized dataset.
    """
    # Common IAGOS to MONETIO mappings
    rename_dict = {
        "lon": "longitude",
        "lat": "latitude",
        "baro_alt": "altitude",
        "air_temp": "temperature",
        "o3": "ozone",
        "co": "carbon_monoxide",
        "h2o": "water_vapor",
        "no": "nitrogen_monoxide",
        "nox": "nitrogen_oxides",
        "gps_lat": "latitude",
        "gps_lon": "longitude",
        "gps_alt": "altitude",
    }

    # Filter rename_dict to only include variables present in ds
    actual_rename = {}
    for k, v in rename_dict.items():
        if k in ds.variables and v not in ds.variables:
            actual_rename[k] = v

    if actual_rename:
        ds = ds.rename(actual_rename)

    # Ensure coordinates are set correctly for PointReader expectations
    for coord in ["time", "latitude", "longitude", "altitude"]:
        if coord in ds.variables and coord not in ds.coords:
            ds = ds.set_coords(coord)

    # Standard units and metadata
    if "ozone" in ds.variables:
        # Check if units are already set, only override if sure
        if ds["ozone"].attrs.get("units") in ["ppb", "ppbv", "1e-9"]:
            ds["ozone"].attrs["units"] = "ppb"
            ds["ozone"].attrs["standard_name"] = "mole_fraction_of_ozone_in_air"

    if "carbon_monoxide" in ds.variables:
        if ds["carbon_monoxide"].attrs.get("units") in ["ppb", "ppbv", "1e-9"]:
            ds["carbon_monoxide"].attrs["units"] = "ppb"
            ds["carbon_monoxide"].attrs["standard_name"] = (
                "mole_fraction_of_carbon_monoxide_in_air"
            )

    return ds

open_dataset(files=None, dates=None, as_xarray=True, lazy=False, **kwargs)

Retrieve and load IAGOS data.

Parameters:

Name Type Description Default
files Union[str, List[str]]

File path, list of paths, or glob pattern.

 None
dates Union[DatetimeIndex, List[datetime], datetime, str]

Dates to retrieve if files are not provided.

 None
as_xarray bool

Whether to return an xarray.Dataset, by default True.

 True
lazy bool

Whether to return a dask-backed object, by default False.

 False
**kwargs dict

Additional arguments passed to the reader and driver.

 {}

Returns:

Type Description
Union[DataFrame, Dataset, DataFrame]

The loaded IAGOS data.
Source code in monetio/readers/iagos.py 
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
def open_dataset(
    self,
    files: str | list[str] | None = None,
    dates: pd.DatetimeIndex | list[datetime.datetime] | datetime.datetime | str | None = None,
    as_xarray: bool = True,
    lazy: bool = False,
    **kwargs,
) -> xr.Dataset | pd.DataFrame | dd.DataFrame:
    """
    Retrieve and load IAGOS data.

    Parameters
    ----------
    files : Union[str, List[str]], optional
        File path, list of paths, or glob pattern.
    dates : Union[pd.DatetimeIndex, List[datetime], datetime, str], optional
        Dates to retrieve if files are not provided.
    as_xarray : bool, optional
        Whether to return an xarray.Dataset, by default True.
    lazy : bool, optional
        Whether to return a dask-backed object, by default False.
    **kwargs : dict
        Additional arguments passed to the reader and driver.

    Returns
    -------
    Union[pd.DataFrame, xr.Dataset, dd.DataFrame]
        The loaded IAGOS data.
    """
    if files is None:
        if dates is None:
            raise ValueError("Must provide either 'files' or 'dates'.")
        files = self.build_urls(dates, **kwargs)

    if not files:
        if as_xarray:
            return xr.Dataset()
        return pd.DataFrame()

    # IAGOS data is NetCDF. We use xr.open_mfdataset for robustness.
    # It handles both single and multiple files, and laziness.
    ds = xr.open_mfdataset(files, combine="nested", concat_dim="time", **kwargs)

    ds = self.harmonize(ds)

    if not as_xarray:
        return ds.to_dataframe().reset_index()

    ds = update_history(ds, "Read IAGOS data using standardized preprocessing.")
    return ds