From 48eeddf8e40c36e9af94500efdd9ff3f979971a6 Mon Sep 17 00:00:00 2001 From: Peter Luenenschloss <peter.luenenschloss@ufz.de> Date: Fri, 23 Apr 2021 09:28:56 +0200 Subject: [PATCH] black applied --- saqc/core/modules/noise.py | 19 ++++++++++--------- saqc/funcs/constants.py | 11 ++++++++--- saqc/funcs/noise.py | 39 ++++++++++++++++++++++++-------------- saqc/lib/tools.py | 28 ++++++++++++++------------- saqc/lib/ts_operators.py | 12 ++++++------ saqc/lib/types.py | 24 +++++++++++------------ 6 files changed, 76 insertions(+), 57 deletions(-) diff --git a/saqc/core/modules/noise.py b/saqc/core/modules/noise.py index f162e5808..bfba8074f 100644 --- a/saqc/core/modules/noise.py +++ b/saqc/core/modules/noise.py @@ -11,14 +11,15 @@ from saqc.lib.types import ColumnName, FreqString, PositiveInt, PositiveFloat class Noise(ModuleBase): - def flagByStatLowPass(self, - field: ColumnName, - stat: Callable[[numpy.array, pd.Series], float], - winsz: FreqString, - thresh: PositiveFloat, - sub_winsz: FreqString = None, - sub_thresh: PositiveFloat = None, - min_periods: PositiveInt = None, - flag: float = BAD + def flagByStatLowPass( + self, + field: ColumnName, + stat: Callable[[numpy.array, pd.Series], float], + winsz: FreqString, + thresh: PositiveFloat, + sub_winsz: FreqString = None, + sub_thresh: PositiveFloat = None, + min_periods: PositiveInt = None, + flag: float = BAD, ) -> SaQC: return self.defer("flagByStatLowPass", locals()) diff --git a/saqc/funcs/constants.py b/saqc/funcs/constants.py index bc311aae0..f4f36cae1 100644 --- a/saqc/funcs/constants.py +++ b/saqc/funcs/constants.py @@ -143,9 +143,14 @@ def flagByVariance( min_periods = int(np.ceil(pd.Timedelta(window) / pd.Timedelta(delta))) window = pd.Timedelta(window) - to_set = statPass(dataseries, lambda x: varQC(x, max_missing, max_consec_missing), - window, thresh, min_periods=min_periods, comparator=operator.lt) + to_set = statPass( + dataseries, + lambda x: varQC(x, max_missing, max_consec_missing), + window, + thresh, + min_periods=min_periods, + comparator=operator.lt, + ) flags[to_set[to_set].index, field] = flag return data, flags - diff --git a/saqc/funcs/noise.py b/saqc/funcs/noise.py index 7bc5404dc..9427eb262 100644 --- a/saqc/funcs/noise.py +++ b/saqc/funcs/noise.py @@ -11,18 +11,20 @@ from saqc.lib.types import ColumnName, FreqString, PositiveInt, PositiveFloat, L from saqc.lib.tools import statPass -@register(masking='field', module="noise") -def flagByStatLowPass(data: DictOfSeries, - field: ColumnName, - flags: Flags, - stat: Callable[[np.array, pd.Series], float], - winsz: FreqString, - thresh: PositiveFloat, - sub_winsz: FreqString = None, - sub_thresh: PositiveFloat = None, - min_periods: PositiveInt = None, - flag: float = BAD, - **kwargs): +@register(masking="field", module="noise") +def flagByStatLowPass( + data: DictOfSeries, + field: ColumnName, + flags: Flags, + stat: Callable[[np.array, pd.Series], float], + winsz: FreqString, + thresh: PositiveFloat, + sub_winsz: FreqString = None, + sub_thresh: PositiveFloat = None, + min_periods: PositiveInt = None, + flag: float = BAD, + **kwargs +): """ Flag *chunks* of length, `winsz`: @@ -61,6 +63,15 @@ def flagByStatLowPass(data: DictOfSeries, winsz = pd.Timedelta(winsz) if sub_winsz: sub_winsz = pd.Timedelta(sub_winsz) - to_set = statPass(datcol, stat, winsz, thresh, sub_winsz, sub_thresh, min_periods, comparator=operator.gt) + to_set = statPass( + datcol, + stat, + winsz, + thresh, + sub_winsz, + sub_thresh, + min_periods, + comparator=operator.gt, + ) flags[to_set[to_set].index, field] = flag - return data, flags \ No newline at end of file + return data, flags diff --git a/saqc/lib/tools.py b/saqc/lib/tools.py index 7816dc985..d5560c37f 100644 --- a/saqc/lib/tools.py +++ b/saqc/lib/tools.py @@ -3,7 +3,7 @@ import re import datetime -from typing import Sequence, Union, Any, Iterator,Callable +from typing import Sequence, Union, Any, Iterator, Callable import operator import itertools import numpy as np @@ -570,7 +570,7 @@ def getFreqDelta(index): return delta -def getAttrOrApply(in_obj, apply_obj, attr_access='__name__', attr_or='apply'): +def getAttrOrApply(in_obj, apply_obj, attr_access="__name__", attr_or="apply"): """ For the repeating task of applying build in (accelerated) methods/funcs (`apply_obj`), of rolling/resampling - like objects (`in_obj`) , @@ -585,14 +585,16 @@ def getAttrOrApply(in_obj, apply_obj, attr_access='__name__', attr_or='apply'): return out -def statPass(datcol: pd.Series, - stat: Callable[[np.array, pd.Series], float], - winsz: pd.Timedelta, - thresh: PositiveFloat, - comparator: Callable[[float, float], bool], - sub_winsz: pd.Timedelta = None, - sub_thresh: PositiveFloat = None, - min_periods: PositiveInt = None): +def statPass( + datcol: pd.Series, + stat: Callable[[np.array, pd.Series], float], + winsz: pd.Timedelta, + thresh: PositiveFloat, + comparator: Callable[[float, float], bool], + sub_winsz: pd.Timedelta = None, + sub_thresh: PositiveFloat = None, + min_periods: PositiveInt = None, +): """ Check `datcol`, if it contains chunks of length `winsz`, exceeding `thresh` with regard to `stat` and `comparator`: @@ -608,13 +610,13 @@ def statPass(datcol: pd.Series, if sub_winsz: stat_sub = datcol.rolling(sub_winsz) stat_sub = getAttrOrApply(stat_sub, stat) - min_stat = stat_sub.rolling(winsz - sub_winsz, closed='both').min() + min_stat = stat_sub.rolling(winsz - sub_winsz, closed="both").min() exceeding_sub = comparator(min_stat, sub_thresh) exceeds = exceeding_sub & exceeds to_set = pd.Series(False, index=exceeds.index) for g in exceeds.groupby(by=exceeds.values): if g[0]: - to_set[g[1].index[0] - winsz:g[1].index[-1]] = True + to_set[g[1].index[0] - winsz : g[1].index[-1]] = True - return to_set \ No newline at end of file + return to_set diff --git a/saqc/lib/ts_operators.py b/saqc/lib/ts_operators.py index af0c67b99..3969229ab 100644 --- a/saqc/lib/ts_operators.py +++ b/saqc/lib/ts_operators.py @@ -361,12 +361,13 @@ def shift2Freq( ) -def butterFilter(x, cutoff, nyq=0.5, filter_order=2, - fill_method='linear', filter_type='low'): +def butterFilter( + x, cutoff, nyq=0.5, filter_order=2, fill_method="linear", filter_type="low" +): """ Applies butterworth filter. `x` is expected to be regularly sampled. - + Parameters ---------- x: pd.Series @@ -377,13 +378,13 @@ def butterFilter(x, cutoff, nyq=0.5, filter_order=2, The niquist-frequency. relates to multiples if the sampling rate. fill_method: Literal[‘nearest’, ‘zero’, ‘slinear’, ‘quadratic’, ‘cubic’, ‘spline’, ‘barycentric’, ‘polynomial’] Any method keyword, accepted by pandas.Series.interpolate. - + Returns ------- """ na_mask = x.isna() - x = x.interpolate(fill_method).interpolate('ffill').interpolate('bfill') + x = x.interpolate(fill_method).interpolate("ffill").interpolate("bfill") b, a = butter(N=filter_order, Wn=cutoff / nyq, btype=filter_type) y = pd.Series(filtfilt(b, a, x), x.index, name=x.name) y[na_mask] = np.nan @@ -493,4 +494,3 @@ def polynomialInterpolation(data, inter_limit=2, inter_order=2): return interpolateNANs( data, "polynomial", inter_limit=inter_limit, order=inter_order ) - diff --git a/saqc/lib/types.py b/saqc/lib/types.py index dced4a5df..34c282914 100644 --- a/saqc/lib/types.py +++ b/saqc/lib/types.py @@ -1,18 +1,18 @@ #! /usr/bin/env python # -*- coding: utf-8 -*- __all__ = [ - 'T', - 'ArrayLike', - 'PandasLike', - 'DiosLikeT', - 'FuncReturnT', - 'FreqString', - 'ColumnName', - 'IntegerWindow', - 'TimestampColumnName', - 'CurveFitter', - 'PositiveFloat', - 'PositiveInt' + "T", + "ArrayLike", + "PandasLike", + "DiosLikeT", + "FuncReturnT", + "FreqString", + "ColumnName", + "IntegerWindow", + "TimestampColumnName", + "CurveFitter", + "PositiveFloat", + "PositiveInt", ] from typing import TypeVar, Union, NewType -- GitLab