How to use 'pandas remove outliers' in Python

37
def filterout_outliers(l_data, l_date):
38
    '''
39
    Return a list with data filtered from outliers
40
41
    :param l_data: list. data to be analyzed
42
    '''
43
    # === old code to filter outliers =======
44
    # Q3 = np.percentile(l_data, 98)
45
    # Q1 = np.percentile(l_data, 2)
46
    # step = (Q3 - Q1) * 1.5
47
    # step = max(3000., step)
48
    # na_val = np.array(l_data)
49
    # na_val = na_val[(na_val >= Q1 - step) & (na_val <= Q3 + step)]
50
    # return na_val
51
    # =======================================
52
    # group by minute
53
    df_filter = pd.Series(np.array(l_date)/60).astype(int)
54
    l_filter = list((df_filter != df_filter.shift()).values)
55
    l_filter[0] = True
56
    l_filter[-1] = True
57
    return np.array(pd.Series(l_data)[l_filter].values)

116
def _handle_outliers(self, p_o):
117
    """ Sets observation probabilities of outliers to uniform if ignore_outliers is set.
118
    Parameters
119
    ----------
120
    p_o : ndarray((T, N))
121
        output probabilities
122
    """
123
    if self.ignore_outliers:
124
        outliers = np.where(p_o.sum(axis=1)==0)[0]
125
        if outliers.size > 0:
126
            p_o[outliers, :] = 1.0
127
            self.found_outliers = True
128
    return p_o