Skip to content
Snippets Groups Projects
Commit eb1a3afb authored by Peter Lünenschloß's avatar Peter Lünenschloß
Browse files

filterFuncs Added

parent 6ae582ff
No related branches found
No related tags found
1 merge request!256Filter funcs
Pipeline #21824 passed with stage
in 1 minute and 38 seconds
......@@ -2,6 +2,7 @@
# -*- coding: utf-8 -*-
from saqc.core.modules.breaks import Breaks
from saqc.core.modules.noise import Noise
from saqc.core.modules.changepoints import ChangePoints
from saqc.core.modules.constants import Constants
from saqc.core.modules.curvefit import Curvefit
......@@ -30,6 +31,7 @@ class FuncModules:
self.flagtools = FlagTools(obj)
self.generic = Generic(obj)
self.interpolation = Interpolation(obj)
self.noise = Noise(obj)
self.outliers = Outliers(obj)
self.pattern = Pattern(obj)
self.resampling = Resampling(obj)
......
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import annotations
import numpy as np
from saqc.constants import BAD
from saqc.core.modules.base import ModuleBase
from saqc.lib.types import FreqString, IntegerWindow, ColumnName
from saqc.lib.types import ColumnName, FreqString, PositiveInt, PositiveFloat
class Noise(ModuleBase):
def flagByVarianceLowPass(self,
field: ColumnName,
wnsz: FreqString,
thresh: PositiveFloat,
sub_wnsz: FreqString = None,
sub_thresh: PositiveFloat = None,
min_periods: PositiveInt = None,
flag: float = BAD
) -> SaQC:
return self.defer("flagByVarianceLowPass", locals())
......@@ -19,3 +19,4 @@ from saqc.funcs.scores import *
from saqc.funcs.tools import *
from saqc.funcs.transformation import *
from saqc.funcs.flagtools import *
from saqc.funcs.noise import *
#! /usr/bin/env python
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from operator import mod
......
#! /usr/bin/env python
# -*- coding: utf-8 -*-
import pandas as pd
from dios import DictOfSeries
from saqc.constants import *
from saqc.core import register, Flags
from saqc.lib.types import ColumnName, FreqString, PositiveInt, PositiveFloat
@register(masking='field', module="noise")
def flagByVarianceLowPass(data: DictOfSeries,
field: ColumnName,
flags: Flags,
wnsz: FreqString,
thresh: PositiveFloat,
sub_wnsz: FreqString = None,
sub_thresh: PositiveFloat = None,
min_periods: PositiveInt = None,
flag: float = BAD,
**kwargs):
datcol = data[field]
if not min_periods:
min_periods = 0
if not sub_thresh:
sub_thresh = thresh
if not sub_wnsz:
sub_wnsz = wnsz
wnsz = pd.Timedelta(wnsz)
sub_wnsz = pd.Timedelta(sub_wnsz)
stat_parent = datcol.rolling(wnsz, min_periods=min_periods).std()
exceeding_parent = stat_parent > thresh
stat_sub = datcol.rolling(sub_wnsz).std()
min_stat = stat_sub.rolling(wnsz - sub_wnsz, closed='both').min()
exceeding_sub = min_stat > sub_thresh
exceeds = exceeding_sub & exceeding_parent
to_set = pd.Series(False, index=exceeds.index)
for g in exceeds.groupby(by=exceeds.values):
if g[0]:
to_set[g[1].index[0] - wnsz:g[1].index[-1]] = True
flags[exceeds[exceeds].index, field] = flag
return data, flags
\ No newline at end of file
......@@ -11,6 +11,8 @@ __all__ = [
'IntegerWindow',
'TimestampColumnName',
'CurveFitter',
'PositiveFloat',
'PositiveInt'
]
from typing import TypeVar, Union, NewType
......@@ -37,6 +39,8 @@ FreqString = NewType("FreqString", Literal["D", "H", "T", "min", "S", "L", "ms",
ColumnName = NewType("ColumnName", str)
IntegerWindow = NewType("IntegerWindow", int)
TimestampColumnName = TypeVar("TimestampColumnName", bound=str)
PositiveFloat = NewType("PositiveFloat", float)
PositiveInt = NewType("PositiveInt", int)
# needed for deeper typy hinting magic
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment