filterFuncs Added

saqc/core/modules/__init__.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 from saqc.core.modules.breaks import Breaks
+from saqc.core.modules.noise import Noise
 from saqc.core.modules.changepoints import ChangePoints
 from saqc.core.modules.constants import Constants
 from saqc.core.modules.curvefit import Curvefit
@@ -30,6 +31,7 @@ class FuncModules:
         self.flagtools = FlagTools(obj)
         self.generic = Generic(obj)
         self.interpolation = Interpolation(obj)
+        self.noise = Noise(obj)
         self.outliers = Outliers(obj)
         self.pattern = Pattern(obj)
         self.resampling = Resampling(obj)

saqc/core/modules/noise.py

+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+from __future__ import annotations
+
+import numpy as np
+
+from saqc.constants import BAD
+from saqc.core.modules.base import ModuleBase
+from saqc.lib.types import FreqString, IntegerWindow, ColumnName
+from saqc.lib.types import ColumnName, FreqString, PositiveInt, PositiveFloat
+
+
+class Noise(ModuleBase):
+    def flagByVarianceLowPass(self,
+                              field: ColumnName,
+                              wnsz: FreqString,
+                              thresh: PositiveFloat,
+                              sub_wnsz: FreqString = None,
+                              sub_thresh: PositiveFloat = None,
+                              min_periods: PositiveInt = None,
+                              flag: float = BAD
+    ) -> SaQC:
+        return self.defer("flagByVarianceLowPass", locals())

saqc/funcs/__init__.py

@@ -19,3 +19,4 @@ from saqc.funcs.scores import *
 from saqc.funcs.tools import *
 from saqc.funcs.transformation import *
 from saqc.funcs.flagtools import *
+from saqc.funcs.noise import *

saqc/funcs/constants.py

-#! /usr/bin/env python
+    #! /usr/bin/env python
 # -*- coding: utf-8 -*-
 
 from operator import mod

saqc/funcs/noise.py

+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+import pandas as pd
+from dios import DictOfSeries
+
+from saqc.constants import *
+from saqc.core import register, Flags
+from saqc.lib.types import ColumnName, FreqString, PositiveInt, PositiveFloat
+
+
+@register(masking='field', module="noise")
+def flagByVarianceLowPass(data: DictOfSeries,
+                          field: ColumnName,
+                          flags: Flags,
+                          wnsz: FreqString,
+                          thresh: PositiveFloat,
+                          sub_wnsz: FreqString = None,
+                          sub_thresh: PositiveFloat = None,
+                          min_periods: PositiveInt = None,
+                          flag: float = BAD,
+                          **kwargs):
+
+    datcol = data[field]
+    if not min_periods:
+        min_periods = 0
+    if not sub_thresh:
+        sub_thresh = thresh
+    if not sub_wnsz:
+        sub_wnsz = wnsz
+
+    wnsz = pd.Timedelta(wnsz)
+    sub_wnsz = pd.Timedelta(sub_wnsz)
+
+    stat_parent = datcol.rolling(wnsz, min_periods=min_periods).std()
+    exceeding_parent = stat_parent > thresh
+    stat_sub = datcol.rolling(sub_wnsz).std()
+
+    min_stat = stat_sub.rolling(wnsz - sub_wnsz, closed='both').min()
+    exceeding_sub = min_stat > sub_thresh
+    exceeds = exceeding_sub & exceeding_parent
+    to_set = pd.Series(False, index=exceeds.index)
+
+    for g in exceeds.groupby(by=exceeds.values):
+        if g[0]:
+            to_set[g[1].index[0] - wnsz:g[1].index[-1]] = True
+
+    flags[exceeds[exceeds].index, field] = flag
+    return data, flags
\ No newline at end of file

saqc/lib/types.py

@@ -11,6 +11,8 @@ __all__ = [
     'IntegerWindow',
     'TimestampColumnName',
     'CurveFitter',
+    'PositiveFloat',
+    'PositiveInt'
 ]
 
 from typing import TypeVar, Union, NewType
@@ -37,6 +39,8 @@ FreqString = NewType("FreqString", Literal["D", "H", "T", "min", "S", "L", "ms",
 ColumnName = NewType("ColumnName", str)
 IntegerWindow = NewType("IntegerWindow", int)
 TimestampColumnName = TypeVar("TimestampColumnName", bound=str)
+PositiveFloat = NewType("PositiveFloat", float)
+PositiveInt = NewType("PositiveInt", int)
 
 
 # needed for deeper typy hinting magic