Skip to content

nesdis_edr_viirs

NESDIS EDR VIIRS Reader

NESDISEDRVIIRSReader

Bases: GriddedReader

Reader for NESDIS EDR VIIRS gridded AOD data. Available via FTP.

Source code in monetio/readers/nesdis_edr_viirs.py 
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
@register_reader("nesdis_edr_viirs")
class NESDISEDRVIIRSReader(GriddedReader):
    """
    Reader for NESDIS EDR VIIRS gridded AOD data.
    Available via FTP.
    """

    def open_dataset(
        self,
        files: str | list[str] = None,
        dates: pd.DatetimeIndex | list[datetime.datetime] | datetime.datetime | str = None,
        resolution: str = "high",
        **kwargs,
    ) -> xr.Dataset:
        """
        Reads NESDIS EDR VIIRS data.

        Parameters
        ----------
        files : str or list[str], optional
            File path(s) or URL(s).
        dates : pd.DatetimeIndex, list, datetime, or str, optional
            Dates to retrieve. If files is None, this is used to build URLs.
        resolution : str, optional
            'high' (0.10 deg) or 'low' (0.25 deg). Default is 'high'.
        **kwargs : dict
            Additional arguments passed to XarrayDriver.open.

        Returns
        -------
        xr.Dataset
            The NESDIS EDR VIIRS dataset.

        Examples
        --------
        >>> reader = NESDISEDRVIIRSReader()
        >>> ds = reader.open_dataset(date="2023-01-01", resolution="high")
        """
        if files is None:
            if dates is None:
                raise ValueError("Either 'files' or 'dates' must be provided.")
            files = self.build_urls(dates, resolution=resolution)

        if "preprocess" not in kwargs:
            kwargs["preprocess"] = partial(nesdis_edr_viirs_preprocess, resolution=resolution)

        if "read_method" not in kwargs:
            kwargs["read_method"] = read_nesdis_edr_binary

        # Forward resolution to read_method
        kwargs["resolution"] = resolution

        ds = super().open_dataset(files, **kwargs)

        # Update history
        ds = update_history(ds, "Read NESDIS EDR VIIRS data.")

        return ds

    def build_urls(
        self,
        dates: pd.DatetimeIndex | list[datetime.datetime] | datetime.datetime | str,
        resolution: str = "high",
    ) -> list[str]:
        """
        Build FTP URLs for NESDIS EDR VIIRS data based on dates.

        Parameters
        ----------
        dates : pd.DatetimeIndex, list, datetime, or str
            Dates to retrieve.
        resolution : str, optional
            'high' or 'low', by default "high".

        Returns
        -------
        list[str]
            List of FTP URLs.

        Examples
        --------
        >>> reader = NESDISEDRVIIRSReader()
        >>> urls = reader.build_urls("2023-01-01", resolution="high")
        """
        if isinstance(dates, str | datetime.datetime | pd.Timestamp):
            dates = pd.DatetimeIndex([pd.to_datetime(dates)])
        else:
            dates = pd.to_datetime(dates)

        server = "ftp.star.nesdis.noaa.gov"
        base_dir = "/pub/smcd/jhuang/npp.viirs.aerosol.data/edraot550"

        urls = []
        for d in dates:
            year = d.strftime("%Y")
            yyyymmdd = d.strftime("%Y%m%d")

            if resolution in {"high", "h", "0.10"}:
                filename = f"npp_aot550_edr_gridded_0.10_{yyyymmdd}.high.bin.gz"
            else:
                filename = f"npp_aot550_edr_gridded_0.25_{yyyymmdd}.high.bin.gz"

            url = f"ftp://{server}{base_dir}/{year}/{filename}"
            urls.append(url)
        return urls

build_urls(dates, resolution='high')

Build FTP URLs for NESDIS EDR VIIRS data based on dates.

Parameters:

Name Type Description Default
dates pd.DatetimeIndex, list, datetime, or str

Dates to retrieve.

 required
resolution str

'high' or 'low', by default "high".

 'high'

Returns:

Type Description
list[str]

List of FTP URLs.

Examples:

>>> reader = NESDISEDRVIIRSReader()
>>> urls = reader.build_urls("2023-01-01", resolution="high")
Source code in monetio/readers/nesdis_edr_viirs.py 
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
def build_urls(
    self,
    dates: pd.DatetimeIndex | list[datetime.datetime] | datetime.datetime | str,
    resolution: str = "high",
) -> list[str]:
    """
    Build FTP URLs for NESDIS EDR VIIRS data based on dates.

    Parameters
    ----------
    dates : pd.DatetimeIndex, list, datetime, or str
        Dates to retrieve.
    resolution : str, optional
        'high' or 'low', by default "high".

    Returns
    -------
    list[str]
        List of FTP URLs.

    Examples
    --------
    >>> reader = NESDISEDRVIIRSReader()
    >>> urls = reader.build_urls("2023-01-01", resolution="high")
    """
    if isinstance(dates, str | datetime.datetime | pd.Timestamp):
        dates = pd.DatetimeIndex([pd.to_datetime(dates)])
    else:
        dates = pd.to_datetime(dates)

    server = "ftp.star.nesdis.noaa.gov"
    base_dir = "/pub/smcd/jhuang/npp.viirs.aerosol.data/edraot550"

    urls = []
    for d in dates:
        year = d.strftime("%Y")
        yyyymmdd = d.strftime("%Y%m%d")

        if resolution in {"high", "h", "0.10"}:
            filename = f"npp_aot550_edr_gridded_0.10_{yyyymmdd}.high.bin.gz"
        else:
            filename = f"npp_aot550_edr_gridded_0.25_{yyyymmdd}.high.bin.gz"

        url = f"ftp://{server}{base_dir}/{year}/{filename}"
        urls.append(url)
    return urls

open_dataset(files=None, dates=None, resolution='high', **kwargs)

Reads NESDIS EDR VIIRS data.

Parameters:

Name Type Description Default
files str or list[str]

File path(s) or URL(s).

 None
dates pd.DatetimeIndex, list, datetime, or str

Dates to retrieve. If files is None, this is used to build URLs.

 None
resolution str

'high' (0.10 deg) or 'low' (0.25 deg). Default is 'high'.

 'high'
**kwargs dict

Additional arguments passed to XarrayDriver.open.

 {}

Returns:

Type Description
Dataset

The NESDIS EDR VIIRS dataset.

Examples:

>>> reader = NESDISEDRVIIRSReader()
>>> ds = reader.open_dataset(date="2023-01-01", resolution="high")
Source code in monetio/readers/nesdis_edr_viirs.py 
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
def open_dataset(
    self,
    files: str | list[str] = None,
    dates: pd.DatetimeIndex | list[datetime.datetime] | datetime.datetime | str = None,
    resolution: str = "high",
    **kwargs,
) -> xr.Dataset:
    """
    Reads NESDIS EDR VIIRS data.

    Parameters
    ----------
    files : str or list[str], optional
        File path(s) or URL(s).
    dates : pd.DatetimeIndex, list, datetime, or str, optional
        Dates to retrieve. If files is None, this is used to build URLs.
    resolution : str, optional
        'high' (0.10 deg) or 'low' (0.25 deg). Default is 'high'.
    **kwargs : dict
        Additional arguments passed to XarrayDriver.open.

    Returns
    -------
    xr.Dataset
        The NESDIS EDR VIIRS dataset.

    Examples
    --------
    >>> reader = NESDISEDRVIIRSReader()
    >>> ds = reader.open_dataset(date="2023-01-01", resolution="high")
    """
    if files is None:
        if dates is None:
            raise ValueError("Either 'files' or 'dates' must be provided.")
        files = self.build_urls(dates, resolution=resolution)

    if "preprocess" not in kwargs:
        kwargs["preprocess"] = partial(nesdis_edr_viirs_preprocess, resolution=resolution)

    if "read_method" not in kwargs:
        kwargs["read_method"] = read_nesdis_edr_binary

    # Forward resolution to read_method
    kwargs["resolution"] = resolution

    ds = super().open_dataset(files, **kwargs)

    # Update history
    ds = update_history(ds, "Read NESDIS EDR VIIRS data.")

    return ds

nesdis_edr_viirs_preprocess(ds, resolution='high')

Preprocess NESDIS EDR VIIRS dataset: generate coordinates and metadata.

Parameters:

Name Type Description Default
ds Dataset

Input dataset.

 required
resolution str

'high' or 'low', by default "high".

 'high'

Returns:

Type Description
Dataset

Processed dataset.

Examples:

>>> ds = nesdis_edr_viirs_preprocess(ds, resolution="high")
Source code in monetio/readers/nesdis_edr_viirs.py 
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
def nesdis_edr_viirs_preprocess(ds: xr.Dataset, resolution: str = "high") -> xr.Dataset:
    """
    Preprocess NESDIS EDR VIIRS dataset: generate coordinates and metadata.

    Parameters
    ----------
    ds : xr.Dataset
        Input dataset.
    resolution : str, optional
        'high' or 'low', by default "high".

    Returns
    -------
    xr.Dataset
        Processed dataset.

    Examples
    --------
    >>> ds = nesdis_edr_viirs_preprocess(ds, resolution="high")
    """
    if resolution in {"high", "h", "0.10"}:
        nlat, nlon = 1800, 3600
    else:
        nlat, nlon = 720, 1440

    # Generate lat/lon coords
    # Centers of the 0.10 or 0.25 degree cells
    lons = np.linspace(-179.875, 179.875, nlon)
    lats = np.linspace(-89.875, 89.875, nlat)

    # Lazy coordinate generation
    lon1d = xr.DataArray(lons, dims=("x",), name="longitude")
    lat1d = xr.DataArray(lats, dims=("y",), name="latitude")
    lat2d, lon2d = xr.broadcast(lat1d, lon1d)

    ds = ds.assign_coords(
        latitude=lat2d.assign_attrs({"units": "degrees_north", "standard_name": "latitude"}),
        longitude=lon2d.assign_attrs({"units": "degrees_east", "standard_name": "longitude"}),
    )

    # Mask invalid values
    # Binary uses -999.9 for missing.
    ds["aod_550"] = ds["aod_550"].where(ds["aod_550"] > -900)

    # Metadata
    ds.aod_550.attrs.update(
        {
            "long_name": "Aerosol Optical Thickness at 550nm",
            "units": "1",
            "standard_name": "atmosphere_optical_thickness_due_to_ambient_aerosol",
        }
    )

    # Re-order dimensions to (time, y, x) for consistency if time exists
    if "time" in ds.dims:
        ds = ds.transpose("time", "y", "x")

    # Provenance
    ds = update_history(
        ds, "Preprocessed NESDIS EDR VIIRS binary data using standardized preprocessing."
    )

    return ds

read_nesdis_edr_binary(fname, **kwargs)

Read NESDIS EDR VIIRS binary data into an xarray.Dataset. Supports streaming from fsspec-compatible files (including .gz).

Parameters:

Name Type Description Default
fname str

Path or URL to the binary file.

 required
**kwargs dict

Additional arguments (resolution, lazy).

 {}

Returns:

Type Description
Dataset

The dataset containing AOD.

Examples:

>>> ds = read_nesdis_edr_binary("npp_aot550_edr_gridded_0.10_20230101.high.bin.gz", resolution="high")
Source code in monetio/readers/nesdis_edr_viirs.py 
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
def read_nesdis_edr_binary(fname: str, **kwargs) -> xr.Dataset:
    """
    Read NESDIS EDR VIIRS binary data into an xarray.Dataset.
    Supports streaming from fsspec-compatible files (including .gz).

    Parameters
    ----------
    fname : str
        Path or URL to the binary file.
    **kwargs : dict
        Additional arguments (resolution, lazy).

    Returns
    -------
    xr.Dataset
        The dataset containing AOD.

    Examples
    --------
    >>> ds = read_nesdis_edr_binary("npp_aot550_edr_gridded_0.10_20230101.high.bin.gz", resolution="high")
    """
    resolution = kwargs.get("resolution", "high")
    # XarrayDriver might pop 'lazy' and set 'chunks'.
    lazy = kwargs.get("lazy", "chunks" in kwargs)

    if resolution in {"high", "h", "0.10"}:
        nlat, nlon = 1800, 3600
    else:
        nlat, nlon = 720, 1440

    def _read_core(filename):
        from .drivers import FileUtility

        fs = FileUtility.get_fs(filename)
        # Using compression='infer' to handle .gz files automatically
        with fs.open(filename, compression="infer") as f:
            # Binary file contains 2 layers (AOD and something else), first is AOD.
            # Using np.frombuffer on the stream is efficient.
            data = np.frombuffer(f.read(), dtype="<f4")
            # Reshape and take first layer
            return data.reshape(2, nlat, nlon)[0, :, :].copy()

    if lazy:
        import dask.array as da
        from dask import delayed

        load_binary = delayed(_read_core)(fname)
        aot = da.from_delayed(load_binary, shape=(nlat, nlon), dtype="<f4")
    else:
        aot = _read_core(fname)

    ds = xr.Dataset(data_vars={"aod_550": (("y", "x"), aot)})

    # Extract time from filename if possible
    # Example: npp_aot550_edr_gridded_0.10_20230101.high.bin.gz
    basename = os.path.basename(fname)
    try:
        # Split by underscore and find the date part (8 digits)
        import re

        match = re.search(r"(\d{8})", basename)
        if match:
            date_str = match.group(1)
            date = pd.to_datetime(date_str)
            ds = ds.assign_coords(time=date).expand_dims("time")
    except (ValueError, TypeError):
        pass

    return ds