Skip to content
Snippets Groups Projects
Commit 48eeddf8 authored by Peter Lünenschloß's avatar Peter Lünenschloß
Browse files

black applied

parent f93c246c
No related branches found
No related tags found
1 merge request!256Filter funcs
Pipeline #22928 passed with stage
in 1 minute and 46 seconds
......@@ -11,14 +11,15 @@ from saqc.lib.types import ColumnName, FreqString, PositiveInt, PositiveFloat
class Noise(ModuleBase):
def flagByStatLowPass(self,
field: ColumnName,
stat: Callable[[numpy.array, pd.Series], float],
winsz: FreqString,
thresh: PositiveFloat,
sub_winsz: FreqString = None,
sub_thresh: PositiveFloat = None,
min_periods: PositiveInt = None,
flag: float = BAD
def flagByStatLowPass(
self,
field: ColumnName,
stat: Callable[[numpy.array, pd.Series], float],
winsz: FreqString,
thresh: PositiveFloat,
sub_winsz: FreqString = None,
sub_thresh: PositiveFloat = None,
min_periods: PositiveInt = None,
flag: float = BAD,
) -> SaQC:
return self.defer("flagByStatLowPass", locals())
......@@ -143,9 +143,14 @@ def flagByVariance(
min_periods = int(np.ceil(pd.Timedelta(window) / pd.Timedelta(delta)))
window = pd.Timedelta(window)
to_set = statPass(dataseries, lambda x: varQC(x, max_missing, max_consec_missing),
window, thresh, min_periods=min_periods, comparator=operator.lt)
to_set = statPass(
dataseries,
lambda x: varQC(x, max_missing, max_consec_missing),
window,
thresh,
min_periods=min_periods,
comparator=operator.lt,
)
flags[to_set[to_set].index, field] = flag
return data, flags
......@@ -11,18 +11,20 @@ from saqc.lib.types import ColumnName, FreqString, PositiveInt, PositiveFloat, L
from saqc.lib.tools import statPass
@register(masking='field', module="noise")
def flagByStatLowPass(data: DictOfSeries,
field: ColumnName,
flags: Flags,
stat: Callable[[np.array, pd.Series], float],
winsz: FreqString,
thresh: PositiveFloat,
sub_winsz: FreqString = None,
sub_thresh: PositiveFloat = None,
min_periods: PositiveInt = None,
flag: float = BAD,
**kwargs):
@register(masking="field", module="noise")
def flagByStatLowPass(
data: DictOfSeries,
field: ColumnName,
flags: Flags,
stat: Callable[[np.array, pd.Series], float],
winsz: FreqString,
thresh: PositiveFloat,
sub_winsz: FreqString = None,
sub_thresh: PositiveFloat = None,
min_periods: PositiveInt = None,
flag: float = BAD,
**kwargs
):
"""
Flag *chunks* of length, `winsz`:
......@@ -61,6 +63,15 @@ def flagByStatLowPass(data: DictOfSeries,
winsz = pd.Timedelta(winsz)
if sub_winsz:
sub_winsz = pd.Timedelta(sub_winsz)
to_set = statPass(datcol, stat, winsz, thresh, sub_winsz, sub_thresh, min_periods, comparator=operator.gt)
to_set = statPass(
datcol,
stat,
winsz,
thresh,
sub_winsz,
sub_thresh,
min_periods,
comparator=operator.gt,
)
flags[to_set[to_set].index, field] = flag
return data, flags
\ No newline at end of file
return data, flags
......@@ -3,7 +3,7 @@
import re
import datetime
from typing import Sequence, Union, Any, Iterator,Callable
from typing import Sequence, Union, Any, Iterator, Callable
import operator
import itertools
import numpy as np
......@@ -570,7 +570,7 @@ def getFreqDelta(index):
return delta
def getAttrOrApply(in_obj, apply_obj, attr_access='__name__', attr_or='apply'):
def getAttrOrApply(in_obj, apply_obj, attr_access="__name__", attr_or="apply"):
"""
For the repeating task of applying build in (accelerated) methods/funcs (`apply_obj`),
of rolling/resampling - like objects (`in_obj`) ,
......@@ -585,14 +585,16 @@ def getAttrOrApply(in_obj, apply_obj, attr_access='__name__', attr_or='apply'):
return out
def statPass(datcol: pd.Series,
stat: Callable[[np.array, pd.Series], float],
winsz: pd.Timedelta,
thresh: PositiveFloat,
comparator: Callable[[float, float], bool],
sub_winsz: pd.Timedelta = None,
sub_thresh: PositiveFloat = None,
min_periods: PositiveInt = None):
def statPass(
datcol: pd.Series,
stat: Callable[[np.array, pd.Series], float],
winsz: pd.Timedelta,
thresh: PositiveFloat,
comparator: Callable[[float, float], bool],
sub_winsz: pd.Timedelta = None,
sub_thresh: PositiveFloat = None,
min_periods: PositiveInt = None,
):
"""
Check `datcol`, if it contains chunks of length `winsz`, exceeding `thresh` with
regard to `stat` and `comparator`:
......@@ -608,13 +610,13 @@ def statPass(datcol: pd.Series,
if sub_winsz:
stat_sub = datcol.rolling(sub_winsz)
stat_sub = getAttrOrApply(stat_sub, stat)
min_stat = stat_sub.rolling(winsz - sub_winsz, closed='both').min()
min_stat = stat_sub.rolling(winsz - sub_winsz, closed="both").min()
exceeding_sub = comparator(min_stat, sub_thresh)
exceeds = exceeding_sub & exceeds
to_set = pd.Series(False, index=exceeds.index)
for g in exceeds.groupby(by=exceeds.values):
if g[0]:
to_set[g[1].index[0] - winsz:g[1].index[-1]] = True
to_set[g[1].index[0] - winsz : g[1].index[-1]] = True
return to_set
\ No newline at end of file
return to_set
......@@ -361,12 +361,13 @@ def shift2Freq(
)
def butterFilter(x, cutoff, nyq=0.5, filter_order=2,
fill_method='linear', filter_type='low'):
def butterFilter(
x, cutoff, nyq=0.5, filter_order=2, fill_method="linear", filter_type="low"
):
"""
Applies butterworth filter.
`x` is expected to be regularly sampled.
Parameters
----------
x: pd.Series
......@@ -377,13 +378,13 @@ def butterFilter(x, cutoff, nyq=0.5, filter_order=2,
The niquist-frequency. relates to multiples if the sampling rate.
fill_method: Literal[‘nearest’, ‘zero’, ‘slinear’, ‘quadratic’, ‘cubic’, ‘spline’, ‘barycentric’, ‘polynomial’]
Any method keyword, accepted by pandas.Series.interpolate.
Returns
-------
"""
na_mask = x.isna()
x = x.interpolate(fill_method).interpolate('ffill').interpolate('bfill')
x = x.interpolate(fill_method).interpolate("ffill").interpolate("bfill")
b, a = butter(N=filter_order, Wn=cutoff / nyq, btype=filter_type)
y = pd.Series(filtfilt(b, a, x), x.index, name=x.name)
y[na_mask] = np.nan
......@@ -493,4 +494,3 @@ def polynomialInterpolation(data, inter_limit=2, inter_order=2):
return interpolateNANs(
data, "polynomial", inter_limit=inter_limit, order=inter_order
)
#! /usr/bin/env python
# -*- coding: utf-8 -*-
__all__ = [
'T',
'ArrayLike',
'PandasLike',
'DiosLikeT',
'FuncReturnT',
'FreqString',
'ColumnName',
'IntegerWindow',
'TimestampColumnName',
'CurveFitter',
'PositiveFloat',
'PositiveInt'
"T",
"ArrayLike",
"PandasLike",
"DiosLikeT",
"FuncReturnT",
"FreqString",
"ColumnName",
"IntegerWindow",
"TimestampColumnName",
"CurveFitter",
"PositiveFloat",
"PositiveInt",
]
from typing import TypeVar, Union, NewType
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment