Skip to content

improve_mod

IMPROVE

Short summary.

Attributes:

Name Type Description
datestr type

Description of attribute datestr.
df type

Description of attribute df.
daily type

Description of attribute daily.
se_states type

Description of attribute se_states.
ne_states type

Description of attribute ne_states.
nc_states type

Description of attribute nc_states.
sc_states type

Description of attribute sc_states.
r_states type

Description of attribute r_states.
p_states type

Description of attribute p_states.
Source code in monetio/obs/improve_mod.py 
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
class IMPROVE:
    """Short summary.

    Attributes
    ----------
    datestr : type
        Description of attribute `datestr`.
    df : type
        Description of attribute `df`.
    daily : type
        Description of attribute `daily`.
    se_states : type
        Description of attribute `se_states`.
    ne_states : type
        Description of attribute `ne_states`.
    nc_states : type
        Description of attribute `nc_states`.
    sc_states : type
        Description of attribute `sc_states`.
    r_states : type
        Description of attribute `r_states`.
    p_states : type
        Description of attribute `p_states`.

    """

    def __init__(self):
        self.datestr = []
        self.df = None
        self.daily = True

    def add_data(self, fname, add_meta=False, delimiter="\t"):
        """This assumes that you have downloaded the data from
                        http://views.cira.colostate.edu/fed/DataWizard/Default.aspx
                The data is the IMPROVE Aerosol dataset
                Any number of sites
                Parameters included are All
                Fields include Dataset,Site,Date,Parameter,POC,Data_value,Unit,
                Latitude,Longitude,State,EPA Site Code Options are delimited
                ',' data only and normalized skinny format

        Parameters
        ----------
        fname : type
            Description of parameter `fname`.
        output : type
            Description of parameter `output` (the default is '').

        Returns
        -------
        type
            Description of returned object.

        """
        from .epa_util import read_monitor_file

        f = open(fname)
        lines = f.readlines()
        skiprows = 0
        skip = False
        for i, line in enumerate(lines):
            if line == "Data\n":
                skip = True
                skiprows = i + 1
                break
        # if meta data is included
        if skip:
            df = pd.read_csv(
                fname,
                delimiter=delimiter,
                parse_dates=[2],
                dtype={"EPACode": str},
                skiprows=skiprows,
            )
        else:
            df = pd.read_csv(
                fname,
                delimiter=delimiter,
                parse_dates=[2],
                dtype={"EPACode": str},
            )
        df.rename(columns={"EPACode": "epaid"}, inplace=True)
        df.rename(columns={"Val": "Obs"}, inplace=True)
        df.rename(columns={"State": "state_name"}, inplace=True)
        df.rename(columns={"ParamCode": "variable"}, inplace=True)
        df.rename(columns={"SiteCode": "siteid"}, inplace=True)
        df.rename(columns={"Unit": "Units"}, inplace=True)
        df.rename(columns={"Date": "time"}, inplace=True)
        df.drop("Dataset", axis=1, inplace=True)
        df["time"] = pd.to_datetime(df.time, format="%Y%m%d")
        df.columns = [i.lower() for i in df.columns]
        if pd.Series(df.keys()).isin(["epaid"]).max():
            df["epaid"] = df.epaid.astype(str).str.zfill(9)
        if add_meta:
            monitor_df = read_monitor_file(network="IMPROVE")  # .drop(
            # dropkeys, axis=1)
            df = df.merge(monitor_df, how="left", left_on="epaid", right_on="siteid")
            df.drop(["siteid_y", "state_name_y"], inplace=True, axis=1)
            df.rename(
                columns={"siteid_x": "siteid", "state_name_x": "state_name"},
                inplace=True,
            )

        try:
            df.obs.loc[df.obs < df.mdl] = nan
        except Exception:
            df.obs.loc[df.obs < -900] = nan
        self.df = df
        return df.copy()

    def load_hdf(self, fname, dates):
        """Short summary.

        Parameters
        ----------
        fname : type
            Description of parameter `fname`.
        dates : type
            Description of parameter `dates`.

        Returns
        -------
        type
            Description of returned object.

        """
        self.df = pd.read_hdf(fname)
        self.get_date_range(self.dates)

    def get_date_range(self, dates):
        """Short summary.

        Parameters
        ----------
        dates : type
            Description of parameter `dates`.

        Returns
        -------
        type
            Description of returned object.

        """
        self.dates = dates
        con = (self.df.time >= dates[0]) & (self.df.time <= dates[-1])
        self.df = self.df.loc[con]

    def set_daterange(self, begin="", end=""):
        """Short summary.

        Parameters
        ----------
        begin : type
            Description of parameter `begin` (the default is '').
        end : type
            Description of parameter `end` (the default is '').

        Returns
        -------
        type
            Description of returned object.

        """
        dates = pd.date_range(start=begin, end=end, freq="h").values.astype("M8[s]").astype("O")
        self.dates = dates

add_data(fname, add_meta=False, delimiter='\t')

This assumes that you have downloaded the data from http://views.cira.colostate.edu/fed/DataWizard/Default.aspx The data is the IMPROVE Aerosol dataset Any number of sites Parameters included are All Fields include Dataset,Site,Date,Parameter,POC,Data_value,Unit, Latitude,Longitude,State,EPA Site Code Options are delimited ',' data only and normalized skinny format

Parameters:

Name Type Description Default
fname type

Description of parameter fname.

 required
output type

Description of parameter output (the default is '').

 required

Returns:

Type Description
type

Description of returned object.
Source code in monetio/obs/improve_mod.py 
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
def add_data(self, fname, add_meta=False, delimiter="\t"):
    """This assumes that you have downloaded the data from
                    http://views.cira.colostate.edu/fed/DataWizard/Default.aspx
            The data is the IMPROVE Aerosol dataset
            Any number of sites
            Parameters included are All
            Fields include Dataset,Site,Date,Parameter,POC,Data_value,Unit,
            Latitude,Longitude,State,EPA Site Code Options are delimited
            ',' data only and normalized skinny format

    Parameters
    ----------
    fname : type
        Description of parameter `fname`.
    output : type
        Description of parameter `output` (the default is '').

    Returns
    -------
    type
        Description of returned object.

    """
    from .epa_util import read_monitor_file

    f = open(fname)
    lines = f.readlines()
    skiprows = 0
    skip = False
    for i, line in enumerate(lines):
        if line == "Data\n":
            skip = True
            skiprows = i + 1
            break
    # if meta data is included
    if skip:
        df = pd.read_csv(
            fname,
            delimiter=delimiter,
            parse_dates=[2],
            dtype={"EPACode": str},
            skiprows=skiprows,
        )
    else:
        df = pd.read_csv(
            fname,
            delimiter=delimiter,
            parse_dates=[2],
            dtype={"EPACode": str},
        )
    df.rename(columns={"EPACode": "epaid"}, inplace=True)
    df.rename(columns={"Val": "Obs"}, inplace=True)
    df.rename(columns={"State": "state_name"}, inplace=True)
    df.rename(columns={"ParamCode": "variable"}, inplace=True)
    df.rename(columns={"SiteCode": "siteid"}, inplace=True)
    df.rename(columns={"Unit": "Units"}, inplace=True)
    df.rename(columns={"Date": "time"}, inplace=True)
    df.drop("Dataset", axis=1, inplace=True)
    df["time"] = pd.to_datetime(df.time, format="%Y%m%d")
    df.columns = [i.lower() for i in df.columns]
    if pd.Series(df.keys()).isin(["epaid"]).max():
        df["epaid"] = df.epaid.astype(str).str.zfill(9)
    if add_meta:
        monitor_df = read_monitor_file(network="IMPROVE")  # .drop(
        # dropkeys, axis=1)
        df = df.merge(monitor_df, how="left", left_on="epaid", right_on="siteid")
        df.drop(["siteid_y", "state_name_y"], inplace=True, axis=1)
        df.rename(
            columns={"siteid_x": "siteid", "state_name_x": "state_name"},
            inplace=True,
        )

    try:
        df.obs.loc[df.obs < df.mdl] = nan
    except Exception:
        df.obs.loc[df.obs < -900] = nan
    self.df = df
    return df.copy()

get_date_range(dates)

Short summary.

Parameters:

Name Type Description Default
dates type

Description of parameter dates.

 required

Returns:

Type Description
type

Description of returned object.
Source code in monetio/obs/improve_mod.py 
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
def get_date_range(self, dates):
    """Short summary.

    Parameters
    ----------
    dates : type
        Description of parameter `dates`.

    Returns
    -------
    type
        Description of returned object.

    """
    self.dates = dates
    con = (self.df.time >= dates[0]) & (self.df.time <= dates[-1])
    self.df = self.df.loc[con]

load_hdf(fname, dates)

Short summary.

Parameters:

Name Type Description Default
fname type

Description of parameter fname.

 required
dates type

Description of parameter dates.

 required

Returns:

Type Description
type

Description of returned object.
Source code in monetio/obs/improve_mod.py 
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
def load_hdf(self, fname, dates):
    """Short summary.

    Parameters
    ----------
    fname : type
        Description of parameter `fname`.
    dates : type
        Description of parameter `dates`.

    Returns
    -------
    type
        Description of returned object.

    """
    self.df = pd.read_hdf(fname)
    self.get_date_range(self.dates)

set_daterange(begin='', end='')

Short summary.

Parameters:

Name Type Description Default
begin type

Description of parameter begin (the default is '').

 ''
end type

Description of parameter end (the default is '').

 ''

Returns:

Type Description
type

Description of returned object.
Source code in monetio/obs/improve_mod.py 
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
def set_daterange(self, begin="", end=""):
    """Short summary.

    Parameters
    ----------
    begin : type
        Description of parameter `begin` (the default is '').
    end : type
        Description of parameter `end` (the default is '').

    Returns
    -------
    type
        Description of returned object.

    """
    dates = pd.date_range(start=begin, end=end, freq="h").values.astype("M8[s]").astype("O")
    self.dates = dates