From 91a5591ebfb284042f1646b914328e2c1f2cf75c Mon Sep 17 00:00:00 2001
From: Peter Luenenschloss <peter.luenenschloss@ufz.de>
Date: Wed, 21 Apr 2021 11:23:45 +0200
Subject: [PATCH] noise tools
---
saqc/core/modules/noise.py | 1 +
saqc/funcs/noise.py | 10 ++++++++--
saqc/funcs/resampling.py | 2 +-
saqc/lib/tools.py | 15 +++++++++++++++
4 files changed, 25 insertions(+), 3 deletions(-)
diff --git a/saqc/core/modules/noise.py b/saqc/core/modules/noise.py
index 527f2562c..009b19a9a 100644
--- a/saqc/core/modules/noise.py
+++ b/saqc/core/modules/noise.py
@@ -13,6 +13,7 @@ from saqc.lib.types import ColumnName, FreqString, PositiveInt, PositiveFloat
class Noise(ModuleBase):
def flagByVarianceLowPass(self,
field: ColumnName,
+ stat: Callable[[numpy.array, pd.Series], float],
wnsz: FreqString,
thresh: PositiveFloat,
sub_wnsz: FreqString = None,
diff --git a/saqc/funcs/noise.py b/saqc/funcs/noise.py
index 57f4745a4..9057f4d56 100644
--- a/saqc/funcs/noise.py
+++ b/saqc/funcs/noise.py
@@ -1,17 +1,21 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
import pandas as pd
+import numpy as np
from dios import DictOfSeries
+from typing import Callable
from saqc.constants import *
from saqc.core import register, Flags
from saqc.lib.types import ColumnName, FreqString, PositiveInt, PositiveFloat
+from saqc.lib.tools import getAttrOrApply
@register(masking='field', module="noise")
def flagByVarianceLowPass(data: DictOfSeries,
field: ColumnName,
flags: Flags,
+ stat: Callable[[np.array, pd.Series], float],
wnsz: FreqString,
thresh: PositiveFloat,
sub_wnsz: FreqString = None,
@@ -31,9 +35,11 @@ def flagByVarianceLowPass(data: DictOfSeries,
wnsz = pd.Timedelta(wnsz)
sub_wnsz = pd.Timedelta(sub_wnsz)
- stat_parent = datcol.rolling(wnsz, min_periods=min_periods).std()
+ stat_parent = datcol.rolling(wnsz, min_periods=min_periods)
+ stat_parent = getAttrOrApply(stat_parent, stat)
exceeding_parent = stat_parent > thresh
- stat_sub = datcol.rolling(sub_wnsz).std()
+ stat_sub = datcol.rolling(sub_wnsz)
+ stat_sub = getAttrOrApply(stat_sub, stat)
min_stat = stat_sub.rolling(wnsz - sub_wnsz, closed='both').min()
exceeding_sub = min_stat > sub_thresh
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 8bb871cdd..36a326388 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -605,7 +605,7 @@ def reindexFlags(
flags: Flags,
method: Literal[
"inverse_fagg", "inverse_bagg", "inverse_nagg",
- "inverse_fshift", "inverse_bshift", "inverse_nshift"
+ "inverse_fshift", "inverse_bshift", "inverse_nshift", "match"
],
source: str,
freq: Optional[str] = None,
diff --git a/saqc/lib/tools.py b/saqc/lib/tools.py
index 1f15df6d0..3fe9c00d7 100644
--- a/saqc/lib/tools.py
+++ b/saqc/lib/tools.py
@@ -532,3 +532,18 @@ def getFreqDelta(index):
if i.equals(index):
return i[1] - i[0]
return delta
+
+
+def getAttrOrApply(in_obj, apply_obj, attr_access='__name__', attr_or='apply'):
+ """
+ For the repeating task of applying build in (accelerated) methods/funcs (`apply_obj`),
+ of rolling/resampling - like objects (`in_obj`) ,
+ if those build-ins are available, or pass the method/func to the objects apply-like method, otherwise.
+
+ """
+ try:
+ out = getattr(in_obj, getattr(apply_obj, attr_access))()
+ except AttributeError:
+ out = getattr(in_obj, attr_or)(apply_obj)
+
+ return out
\ No newline at end of file
--
GitLab