From 91a5591ebfb284042f1646b914328e2c1f2cf75c Mon Sep 17 00:00:00 2001 From: Peter Luenenschloss <peter.luenenschloss@ufz.de> Date: Wed, 21 Apr 2021 11:23:45 +0200 Subject: [PATCH] noise tools --- saqc/core/modules/noise.py | 1 + saqc/funcs/noise.py | 10 ++++++++-- saqc/funcs/resampling.py | 2 +- saqc/lib/tools.py | 15 +++++++++++++++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/saqc/core/modules/noise.py b/saqc/core/modules/noise.py index 527f2562c..009b19a9a 100644 --- a/saqc/core/modules/noise.py +++ b/saqc/core/modules/noise.py @@ -13,6 +13,7 @@ from saqc.lib.types import ColumnName, FreqString, PositiveInt, PositiveFloat class Noise(ModuleBase): def flagByVarianceLowPass(self, field: ColumnName, + stat: Callable[[numpy.array, pd.Series], float], wnsz: FreqString, thresh: PositiveFloat, sub_wnsz: FreqString = None, diff --git a/saqc/funcs/noise.py b/saqc/funcs/noise.py index 57f4745a4..9057f4d56 100644 --- a/saqc/funcs/noise.py +++ b/saqc/funcs/noise.py @@ -1,17 +1,21 @@ #! /usr/bin/env python # -*- coding: utf-8 -*- import pandas as pd +import numpy as np from dios import DictOfSeries +from typing import Callable from saqc.constants import * from saqc.core import register, Flags from saqc.lib.types import ColumnName, FreqString, PositiveInt, PositiveFloat +from saqc.lib.tools import getAttrOrApply @register(masking='field', module="noise") def flagByVarianceLowPass(data: DictOfSeries, field: ColumnName, flags: Flags, + stat: Callable[[np.array, pd.Series], float], wnsz: FreqString, thresh: PositiveFloat, sub_wnsz: FreqString = None, @@ -31,9 +35,11 @@ def flagByVarianceLowPass(data: DictOfSeries, wnsz = pd.Timedelta(wnsz) sub_wnsz = pd.Timedelta(sub_wnsz) - stat_parent = datcol.rolling(wnsz, min_periods=min_periods).std() + stat_parent = datcol.rolling(wnsz, min_periods=min_periods) + stat_parent = getAttrOrApply(stat_parent, stat) exceeding_parent = stat_parent > thresh - stat_sub = datcol.rolling(sub_wnsz).std() + stat_sub = datcol.rolling(sub_wnsz) + stat_sub = getAttrOrApply(stat_sub, stat) min_stat = stat_sub.rolling(wnsz - sub_wnsz, closed='both').min() exceeding_sub = min_stat > sub_thresh diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py index 8bb871cdd..36a326388 100644 --- a/saqc/funcs/resampling.py +++ b/saqc/funcs/resampling.py @@ -605,7 +605,7 @@ def reindexFlags( flags: Flags, method: Literal[ "inverse_fagg", "inverse_bagg", "inverse_nagg", - "inverse_fshift", "inverse_bshift", "inverse_nshift" + "inverse_fshift", "inverse_bshift", "inverse_nshift", "match" ], source: str, freq: Optional[str] = None, diff --git a/saqc/lib/tools.py b/saqc/lib/tools.py index 1f15df6d0..3fe9c00d7 100644 --- a/saqc/lib/tools.py +++ b/saqc/lib/tools.py @@ -532,3 +532,18 @@ def getFreqDelta(index): if i.equals(index): return i[1] - i[0] return delta + + +def getAttrOrApply(in_obj, apply_obj, attr_access='__name__', attr_or='apply'): + """ + For the repeating task of applying build in (accelerated) methods/funcs (`apply_obj`), + of rolling/resampling - like objects (`in_obj`) , + if those build-ins are available, or pass the method/func to the objects apply-like method, otherwise. + + """ + try: + out = getattr(in_obj, getattr(apply_obj, attr_access))() + except AttributeError: + out = getattr(in_obj, attr_or)(apply_obj) + + return out \ No newline at end of file -- GitLab