diff --git a/ressources/data/config.csv b/ressources/data/config.csv
index 7bbdbbfbc1765178ac249663c80653fd191a89db..e23612fb9682a0a5e9344967f2b15f65f6d64ab3 100644
--- a/ressources/data/config.csv
+++ b/ressources/data/config.csv
@@ -1,6 +1,6 @@
varname ; test
#----------;---------------------------------------
-SM2 ; resampling.shift(freq="15Min")
-SM2 ; breaks.flagMissing()
-'SM(1|2)+' ; outliers.flagRange(min=10, max=60)
-SM2 ; outliers.flagMAD(window="30d", z=3.5)
+SM2 ; shift(freq="15Min")
+SM2 ; flagMissing()
+'SM(1|2)+' ; flagRange(min=10, max=60)
+SM2 ; flagMAD(window="30d", z=3.5)
diff --git a/ressources/data/config_ci.csv b/ressources/data/config_ci.csv
index ecbe227e8562bdaa967785e1ff538c224b5f4fa4..e5b883102b8a541fc9fc89e6e1ffb7c8f4e1ca55 100644
--- a/ressources/data/config_ci.csv
+++ b/ressources/data/config_ci.csv
@@ -1,8 +1,8 @@
varname ; test
#-------; -----------------------------------------------------
-SM2 ; resampling.shift(freq="15Min")
-'.*' ; outliers.flagRange(min=10, max=60)
-SM2 ; breaks.flagMissing()
-SM2 ; outliers.flagRange(min=10, max=60)
-SM2 ; outliers.flagMAD(window="30d", z=3.5)
-Dummy ; generic.flag(func=(isflagged(SM1) | isflagged(SM2)))
+SM2 ; shift(freq="15Min")
+'.*' ; flagRange(min=10, max=60)
+SM2 ; flagMissing()
+SM2 ; flagRange(min=10, max=60)
+SM2 ; flagMAD(window="30d", z=3.5)
+Dummy ; genericFlag(func=(isflagged(SM1) | isflagged(SM2)))
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 252dca3515687e169146f4bddb6a417599754d47..6b729178193a6193d11365db365f8a57c0ed6296 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -14,7 +14,7 @@ from dios import DictOfSeries, to_dios
from saqc.core.flags import initFlagsLike, Flags
from saqc.core.register import FUNC_MAP
-from saqc.core.modules import FuncModules
+from saqc.core.modules import FunctionsMixin
from saqc.core.translator.basetranslator import Translator, FloatTranslator
from saqc.lib.tools import toSequence
from saqc.lib.types import (
@@ -106,14 +106,13 @@ class Accessor:
return self._obj.__repr__()
-class SaQC(FuncModules):
+class SaQC(FunctionsMixin):
def __init__(
self,
data,
flags=None,
scheme: Translator = None,
):
- super().__init__(self)
data, flags = _prepInput(data, flags)
self._data = data
self._flags = self._initFlags(data, flags)
@@ -235,7 +234,7 @@ class SaQC(FuncModules):
for field, target in zip(fields, targets):
if field != target:
out = out._callFunction(
- FUNC_MAP["tools.copy"],
+ FUNC_MAP["copyField"],
data=out._data,
flags=out._flags,
field=field,
diff --git a/saqc/core/modules/__init__.py b/saqc/core/modules/__init__.py
index 178f57d7c3e5e1427df401024f15dd758898f18a..c3e7f09a588c2f7be9c3f75c00eaba864fcfc4b8 100644
--- a/saqc/core/modules/__init__.py
+++ b/saqc/core/modules/__init__.py
@@ -1,6 +1,5 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-
from saqc.core.modules.breaks import Breaks
from saqc.core.modules.noise import Noise
from saqc.core.modules.changepoints import ChangePoints
@@ -18,25 +17,29 @@ from saqc.core.modules.rolling import Rolling
from saqc.core.modules.scores import Scores
from saqc.core.modules.tools import Tools
from saqc.core.modules.transformation import Transformation
+from saqc.core.register import FUNC_MAP
-class FuncModules:
- def __init__(self, obj):
- # our testing modules
- self.breaks = Breaks(obj)
- self.changepoints = ChangePoints(obj)
- self.constants = Constants(obj)
- self.curvefit = Curvefit(obj)
- self.drift = Drift(obj)
- self.flagtools = FlagTools(obj)
- self.generic = Generic(obj)
- self.interpolation = Interpolation(obj)
- self.noise = Noise(obj)
- self.outliers = Outliers(obj)
- self.pattern = Pattern(obj)
- self.resampling = Resampling(obj)
- self.residues = Residues(obj)
- self.rolling = Rolling(obj)
- self.scores = Scores(obj)
- self.tools = Tools(obj)
- self.transformation = Transformation(obj)
+class FunctionsMixin(
+ Breaks,
+ Noise,
+ ChangePoints,
+ Constants,
+ Curvefit,
+ Drift,
+ FlagTools,
+ Generic,
+ Interpolation,
+ Outliers,
+ Pattern,
+ Resampling,
+ Residues,
+ Rolling,
+ Scores,
+ Tools,
+ Transformation,
+):
+ def _defer(self, fname, flocals):
+ flocals.pop("self", None)
+ fkwargs = flocals.pop("kwargs", {})
+ return self._wrap(FUNC_MAP[fname])(**flocals, **fkwargs)
diff --git a/saqc/core/modules/base.py b/saqc/core/modules/base.py
deleted file mode 100644
index 0f357c622f0f5d737a683ea916dac67bddc2f626..0000000000000000000000000000000000000000
--- a/saqc/core/modules/base.py
+++ /dev/null
@@ -1,20 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from saqc.core.register import FUNC_MAP
-
-
-class ModuleBase:
- def __init__(self, obj):
- self.obj = obj
-
- def __str__(self):
- return self.__class__.__name__.lower()
-
- def __getattr__(self, key):
- raise AttributeError(f"'SaQC.{self}' object has no attribute '{key}'")
-
- def defer(self, fname, flocals):
- flocals.pop("self", None)
- fkwargs = flocals.pop("kwargs", {})
- return self.obj._wrap(FUNC_MAP[f"{self}.{fname}"])(**flocals, **fkwargs)
diff --git a/saqc/core/modules/breaks.py b/saqc/core/modules/breaks.py
index c1cba85907734cc41a899b6f8569b05b317f286f..6def4f2da89f6a5234741b9c5b3fa13e0044c3b0 100644
--- a/saqc/core/modules/breaks.py
+++ b/saqc/core/modules/breaks.py
@@ -3,16 +3,15 @@
from __future__ import annotations
from saqc.constants import BAD, UNFLAGGED
-from saqc.core.modules.base import ModuleBase
import saqc
from saqc.lib.types import FreqString
-class Breaks(ModuleBase):
+class Breaks:
def flagMissing(
self, field: str, flag: float = BAD, to_mask: float = UNFLAGGED, **kwargs
) -> saqc.SaQC:
- return self.defer("flagMissing", locals())
+ return self._defer("flagMissing", locals())
def flagIsolated(
self,
@@ -22,7 +21,7 @@ class Breaks(ModuleBase):
flag: float = BAD,
**kwargs
) -> saqc.SaQC:
- return self.defer("flagIsolated", locals())
+ return self._defer("flagIsolated", locals())
def flagJumps(
self,
@@ -33,4 +32,4 @@ class Breaks(ModuleBase):
flag: float = BAD,
**kwargs
) -> saqc.SaQC:
- return self.defer("flagJumps", locals())
+ return self._defer("flagJumps", locals())
diff --git a/saqc/core/modules/changepoints.py b/saqc/core/modules/changepoints.py
index d4893a67573a84e7f71809226793ee345fa49359..a9a350bf3e12e44763b4ac6ad7cab4078171b37f 100644
--- a/saqc/core/modules/changepoints.py
+++ b/saqc/core/modules/changepoints.py
@@ -8,12 +8,11 @@ import numpy as np
from typing_extensions import Literal
from saqc.constants import BAD
-from saqc.core.modules.base import ModuleBase
import saqc
from saqc.lib.types import FreqString
-class ChangePoints(ModuleBase):
+class ChangePoints:
def flagChangePoints(
self,
field: str,
@@ -27,7 +26,7 @@ class ChangePoints(ModuleBase):
flag: float = BAD,
**kwargs,
) -> saqc.SaQC:
- return self.defer("flagChangePoints", locals())
+ return self._defer("flagChangePoints", locals())
def assignChangePointCluster(
self,
@@ -47,4 +46,4 @@ class ChangePoints(ModuleBase):
flag: float = BAD,
**kwargs,
) -> saqc.SaQC:
- return self.defer("assignChangePointCluster", locals())
+ return self._defer("assignChangePointCluster", locals())
diff --git a/saqc/core/modules/constants.py b/saqc/core/modules/constants.py
index dd38ee776c0973d4d649fe85dd8ad511eb18329c..b11c62cd3a00b1d6e9eef2524c716a9360c76bdf 100644
--- a/saqc/core/modules/constants.py
+++ b/saqc/core/modules/constants.py
@@ -3,12 +3,11 @@
from __future__ import annotations
from saqc.constants import BAD
-from saqc.core.modules.base import ModuleBase
import saqc
from saqc.lib.types import FreqString
-class Constants(ModuleBase):
+class Constants:
def flagByVariance(
self,
field: str,
@@ -19,9 +18,9 @@ class Constants(ModuleBase):
flag: float = BAD,
**kwargs
) -> saqc.SaQC:
- return self.defer("flagByVariance", locals())
+ return self._defer("flagByVariance", locals())
def flagConstants(
self, field: str, thresh: float, window: FreqString, flag: float = BAD, **kwargs
) -> saqc.SaQC:
- return self.defer("flagConstants", locals())
+ return self._defer("flagConstants", locals())
diff --git a/saqc/core/modules/curvefit.py b/saqc/core/modules/curvefit.py
index 70074eb53e16c2b907a41a01beccd6a22c25b322..91a2cd6b7ea17ca43b0c99e492ad6b6a48e84741 100644
--- a/saqc/core/modules/curvefit.py
+++ b/saqc/core/modules/curvefit.py
@@ -7,11 +7,10 @@ from dios import DictOfSeries
from typing_extensions import Literal
from saqc.constants import BAD
-from saqc.core.modules.base import ModuleBase
import saqc
-class Curvefit(ModuleBase):
+class Curvefit:
def fitPolynomial(
self,
field: str,
@@ -23,4 +22,4 @@ class Curvefit(ModuleBase):
flag: float = BAD,
**kwargs
) -> saqc.SaQC:
- return self.defer("fitPolynomial", locals())
+ return self._defer("fitPolynomial", locals())
diff --git a/saqc/core/modules/drift.py b/saqc/core/modules/drift.py
index b1812b23c8177a1ee2547163bf1aa95e05f8c06d..ed248633066cebbdf8f379625c0b51376802c2e4 100644
--- a/saqc/core/modules/drift.py
+++ b/saqc/core/modules/drift.py
@@ -8,13 +8,12 @@ import numpy as np
from scipy.spatial.distance import pdist
from saqc.constants import BAD
-from saqc.core.modules.base import ModuleBase
import saqc
from saqc.funcs import LinkageString
from saqc.lib.types import FreqString, CurveFitter
-class Drift(ModuleBase):
+class Drift:
def flagDriftFromNorm(
self,
field: str,
@@ -30,7 +29,7 @@ class Drift(ModuleBase):
flag: float = BAD,
**kwargs
) -> saqc.SaQC:
- return self.defer("flagDriftFromNorm", locals())
+ return self._defer("flagDriftFromNorm", locals())
def flagDriftFromReference(
self,
@@ -45,7 +44,7 @@ class Drift(ModuleBase):
flag: float = BAD,
**kwargs
) -> saqc.SaQC:
- return self.defer("flagDriftFromReference", locals())
+ return self._defer("flagDriftFromReference", locals())
def flagDriftFromScaledNorm(
self,
@@ -63,7 +62,7 @@ class Drift(ModuleBase):
flag: float = BAD,
**kwargs
) -> saqc.SaQC:
- return self.defer("flagDriftFromScaledNorm", locals())
+ return self._defer("flagDriftFromScaledNorm", locals())
def correctDrift(
self,
@@ -74,7 +73,7 @@ class Drift(ModuleBase):
flag: float = BAD,
**kwargs
) -> saqc.SaQC:
- return self.defer("correctDrift", locals())
+ return self._defer("correctDrift", locals())
def correctRegimeAnomaly(
self,
@@ -85,7 +84,7 @@ class Drift(ModuleBase):
epoch: bool = False,
**kwargs
) -> saqc.SaQC:
- return self.defer("correctRegimeAnomaly", locals())
+ return self._defer("correctRegimeAnomaly", locals())
def correctOffset(
self,
@@ -97,4 +96,4 @@ class Drift(ModuleBase):
tolerance: Optional[FreqString] = None,
**kwargs
) -> saqc.SaQC:
- return self.defer("correctOffset", locals())
+ return self._defer("correctOffset", locals())
diff --git a/saqc/core/modules/flagtools.py b/saqc/core/modules/flagtools.py
index 0c91c0ee5e6e5469966aa0a8363970c949b937d3..a782d9700c1e869a13d6d12d2a6d95ba4dd6c1b7 100644
--- a/saqc/core/modules/flagtools.py
+++ b/saqc/core/modules/flagtools.py
@@ -9,22 +9,21 @@ from dios import DictOfSeries
from typing_extensions import Literal
from saqc.constants import BAD
-from saqc.core.modules.base import ModuleBase
import saqc
-class FlagTools(ModuleBase):
+class FlagTools:
def clearFlags(self, field: str, **kwargs) -> saqc.SaQC:
- return self.defer("clearFlags", locals())
+ return self._defer("clearFlags", locals())
def forceFlags(self, field: str, flag: float = BAD, **kwargs) -> saqc.SaQC:
- return self.defer("forceFlags", locals())
+ return self._defer("forceFlags", locals())
def flagDummy(self, field: str, **kwargs) -> saqc.SaQC:
- return self.defer("flagDummy", locals())
+ return self._defer("flagDummy", locals())
def flagUnflagged(self, field: str, flag: float = BAD, **kwargs) -> saqc.SaQC:
- return self.defer("flagUnflagged", locals())
+ return self._defer("flagUnflagged", locals())
def flagManual(
self,
@@ -35,4 +34,4 @@ class FlagTools(ModuleBase):
flag: float = BAD,
**kwargs,
) -> saqc.SaQC:
- return self.defer("flagManual", locals())
+ return self._defer("flagManual", locals())
diff --git a/saqc/core/modules/generic.py b/saqc/core/modules/generic.py
index 74f6ee89d111ffee1c67a3120f07b854b3eed44e..898be936416168d97c0480e7cd73a687532dfa7b 100644
--- a/saqc/core/modules/generic.py
+++ b/saqc/core/modules/generic.py
@@ -8,21 +8,20 @@ import numpy as np
import pandas as pd
from saqc.constants import UNFLAGGED, BAD
-from saqc.core.modules.base import ModuleBase
import saqc
-class Generic(ModuleBase):
- def process(
+class Generic:
+ def genericProcess(
self,
field: str,
func: Callable[[pd.Series], pd.Series],
to_mask: float = UNFLAGGED,
**kwargs,
) -> saqc.SaQC:
- return self.defer("process", locals())
+ return self._defer("genericProcess", locals())
- def flag(
+ def genericFlag(
self,
field: str,
func: Callable[[pd.Series], pd.Series],
@@ -30,4 +29,4 @@ class Generic(ModuleBase):
to_mask: float = UNFLAGGED,
**kwargs,
) -> saqc.SaQC:
- return self.defer("flag", locals())
+ return self._defer("genericFlag", locals())
diff --git a/saqc/core/modules/interpolation.py b/saqc/core/modules/interpolation.py
index 02b037c78655ed3cb2e0bed48daaa533465704a2..8787bd3250b3438702186cf1c7eb6c7b2200e169 100644
--- a/saqc/core/modules/interpolation.py
+++ b/saqc/core/modules/interpolation.py
@@ -8,12 +8,11 @@ import numpy as np
import pandas as pd
from saqc.constants import UNFLAGGED
-from saqc.core.modules.base import ModuleBase
import saqc
from saqc.funcs.interpolation import _SUPPORTED_METHODS
-class Interpolation(ModuleBase):
+class Interpolation:
def interpolateByRolling(
self,
field: str,
@@ -24,7 +23,7 @@ class Interpolation(ModuleBase):
flag: float = UNFLAGGED,
**kwargs
) -> saqc.SaQC:
- return self.defer("interpolateByRolling", locals())
+ return self._defer("interpolateByRolling", locals())
def interpolateInvalid(
self,
@@ -36,7 +35,7 @@ class Interpolation(ModuleBase):
flag: float = UNFLAGGED,
**kwargs
) -> saqc.SaQC:
- return self.defer("interpolateInvalid", locals())
+ return self._defer("interpolateInvalid", locals())
def interpolateIndex(
self,
@@ -48,4 +47,4 @@ class Interpolation(ModuleBase):
downgrade: bool = False,
**kwargs
) -> saqc.SaQC:
- return self.defer("interpolateIndex", locals())
+ return self._defer("interpolateIndex", locals())
diff --git a/saqc/core/modules/noise.py b/saqc/core/modules/noise.py
index 2b60c617eb07ace6d081d5bb19a2f003d807326b..8985f56f2eae96d36f8df749092f26c9ec54f54e 100644
--- a/saqc/core/modules/noise.py
+++ b/saqc/core/modules/noise.py
@@ -7,12 +7,11 @@ import pandas as pd
from typing import Callable
from saqc.constants import BAD
-from saqc.core.modules.base import ModuleBase
import saqc
from saqc.lib.types import FreqString
-class Noise(ModuleBase):
+class Noise:
def flagByStatLowPass(
self,
field: str,
@@ -25,4 +24,4 @@ class Noise(ModuleBase):
flag: float = BAD,
**kwargs
) -> saqc.SaQC:
- return self.defer("flagByStatLowPass", locals())
+ return self._defer("flagByStatLowPass", locals())
diff --git a/saqc/core/modules/outliers.py b/saqc/core/modules/outliers.py
index 21d5cbb5480b254fe29ddd6339e9ac162ae41702..8e64e99d34a8ce0ee22b12b40b8394263d9cc4a6 100644
--- a/saqc/core/modules/outliers.py
+++ b/saqc/core/modules/outliers.py
@@ -9,12 +9,11 @@ import pandas as pd
from typing_extensions import Literal
from saqc.constants import BAD
-from saqc.core.modules.base import ModuleBase
import saqc
from saqc.lib.types import FreqString
-class Outliers(ModuleBase):
+class Outliers:
def flagByStray(
self,
field: str,
@@ -25,7 +24,7 @@ class Outliers(ModuleBase):
flag: float = BAD,
**kwargs,
) -> saqc.SaQC:
- return self.defer("flagByStray", locals())
+ return self._defer("flagByStray", locals())
def flagMVScores(
self,
@@ -45,7 +44,7 @@ class Outliers(ModuleBase):
flag: float = BAD,
**kwargs,
) -> saqc.SaQC:
- return self.defer("flagMVScores", locals())
+ return self._defer("flagMVScores", locals())
def flagRaise(
self,
@@ -60,7 +59,7 @@ class Outliers(ModuleBase):
flag: float = BAD,
**kwargs,
) -> saqc.SaQC:
- return self.defer("flagRaise", locals())
+ return self._defer("flagRaise", locals())
def flagMAD(
self,
@@ -70,7 +69,7 @@ class Outliers(ModuleBase):
flag: float = BAD,
**kwargs,
) -> saqc.SaQC:
- return self.defer("flagMAD", locals())
+ return self._defer("flagMAD", locals())
def flagOffset(
self,
@@ -82,7 +81,7 @@ class Outliers(ModuleBase):
flag: float = BAD,
**kwargs,
) -> saqc.SaQC:
- return self.defer("flagOffset", locals())
+ return self._defer("flagOffset", locals())
def flagByGrubbs(
self,
@@ -94,7 +93,7 @@ class Outliers(ModuleBase):
flag: float = BAD,
**kwargs,
) -> saqc.SaQC:
- return self.defer("flagByGrubbs", locals())
+ return self._defer("flagByGrubbs", locals())
def flagRange(
self,
@@ -104,7 +103,7 @@ class Outliers(ModuleBase):
flag: float = BAD,
**kwargs,
) -> saqc.SaQC:
- return self.defer("flagRange", locals())
+ return self._defer("flagRange", locals())
def flagCrossStatistic(
self,
@@ -115,4 +114,4 @@ class Outliers(ModuleBase):
flag: float = BAD,
**kwargs,
) -> saqc.SaQC:
- return self.defer("flagCrossStatistic", locals())
+ return self._defer("flagCrossStatistic", locals())
diff --git a/saqc/core/modules/pattern.py b/saqc/core/modules/pattern.py
index dd5ba110661e0a549e8e68f5028fd4f5a5e0d72f..2dbb020bec70fa15b20c44d21f2e8c92edbea4ff 100644
--- a/saqc/core/modules/pattern.py
+++ b/saqc/core/modules/pattern.py
@@ -5,11 +5,10 @@ from __future__ import annotations
from typing import Sequence
from saqc.constants import BAD
-from saqc.core.modules.base import ModuleBase
import saqc
-class Pattern(ModuleBase):
+class Pattern:
def flagPatternByDTW(
self,
field,
@@ -20,9 +19,9 @@ class Pattern(ModuleBase):
flag=BAD,
**kwargs
) -> saqc.SaQC:
- return self.defer("flagPatternByDTW", locals())
+ return self._defer("flagPatternByDTW", locals())
def flagPatternByWavelet(
self, field, reference, widths=(1, 2, 4, 8), waveform="mexh", flag=BAD, **kwargs
) -> saqc.SaQC:
- return self.defer("flagPatternByWavelet", locals())
+ return self._defer("flagPatternByWavelet", locals())
diff --git a/saqc/core/modules/resampling.py b/saqc/core/modules/resampling.py
index 568634403989cd17d9eb73d045e4e7f744f08abd..14350a1b9a34646528d5f6512819576035de8fea 100644
--- a/saqc/core/modules/resampling.py
+++ b/saqc/core/modules/resampling.py
@@ -9,14 +9,13 @@ import pandas as pd
from typing_extensions import Literal
from saqc.constants import BAD
-from saqc.core.modules.base import ModuleBase
import saqc
from saqc.funcs.interpolation import _SUPPORTED_METHODS
-class Resampling(ModuleBase):
+class Resampling:
def linear(self, field: str, freq: str, **kwargs) -> saqc.SaQC:
- return self.defer("linear", locals())
+ return self._defer("linear", locals())
def interpolate(
self,
@@ -26,7 +25,7 @@ class Resampling(ModuleBase):
order: int = 1,
**kwargs,
) -> saqc.SaQC:
- return self.defer("interpolate", locals())
+ return self._defer("interpolate", locals())
def shift(
self,
@@ -36,7 +35,7 @@ class Resampling(ModuleBase):
freq_check: Optional[Literal["check", "auto"]] = None,
**kwargs,
) -> saqc.SaQC:
- return self.defer("shift", locals())
+ return self._defer("shift", locals())
def resample(
self,
@@ -52,7 +51,7 @@ class Resampling(ModuleBase):
freq_check: Optional[Literal["check", "auto"]] = None,
**kwargs,
) -> saqc.SaQC:
- return self.defer("resample", locals())
+ return self._defer("resample", locals())
def reindexFlags(
self,
@@ -71,4 +70,4 @@ class Resampling(ModuleBase):
drop: Optional[bool] = False,
**kwargs,
) -> saqc.SaQC:
- return self.defer("reindexFlags", locals())
+ return self._defer("reindexFlags", locals())
diff --git a/saqc/core/modules/residues.py b/saqc/core/modules/residues.py
index 9774e8b8a4630264e57a76de2b81ec0bc5d4a6e4..d2ee8db46a0217c386360ed0ed8831aa2efaa7d2 100644
--- a/saqc/core/modules/residues.py
+++ b/saqc/core/modules/residues.py
@@ -8,11 +8,10 @@ import numpy as np
from typing_extensions import Literal
from saqc.constants import BAD
-from saqc.core.modules.base import ModuleBase
import saqc
-class Residues(ModuleBase):
+class Residues:
def calculatePolynomialResidues(
self,
field: str,
@@ -23,7 +22,7 @@ class Residues(ModuleBase):
flag: float = BAD,
**kwargs
) -> saqc.SaQC:
- return self.defer("calculatePolynomialResidues", locals())
+ return self._defer("calculatePolynomialResidues", locals())
def calculateRollingResidues(
self,
@@ -36,4 +35,4 @@ class Residues(ModuleBase):
flag: float = BAD,
**kwargs
) -> saqc.SaQC:
- return self.defer("calculateRollingResidues", locals())
+ return self._defer("calculateRollingResidues", locals())
diff --git a/saqc/core/modules/rolling.py b/saqc/core/modules/rolling.py
index de232c66b9640d42c2a73dcd031fcc906f29871a..11131514b158ce4d62a83fe145b49a6e8ef864e3 100644
--- a/saqc/core/modules/rolling.py
+++ b/saqc/core/modules/rolling.py
@@ -7,11 +7,9 @@ import numpy as np
import pandas as pd
from saqc.constants import BAD
-from saqc.core.modules.base import ModuleBase
-import saqc
-class Rolling(ModuleBase):
+class Rolling:
def roll(
self,
field: str,
@@ -24,4 +22,4 @@ class Rolling(ModuleBase):
flag: float = BAD,
**kwargs
):
- return self.defer("roll", locals())
+ return self._defer("roll", locals())
diff --git a/saqc/core/modules/scores.py b/saqc/core/modules/scores.py
index 1d119e1516464026f6dd14c7ddc719ef5fd33b8f..cc015d98fc63a466f22346cfd174a3402362460d 100644
--- a/saqc/core/modules/scores.py
+++ b/saqc/core/modules/scores.py
@@ -8,12 +8,10 @@ import numpy as np
import pandas as pd
from typing_extensions import Literal
-from saqc.core import Flags
-from saqc.core.modules.base import ModuleBase
import saqc
-class Scores(ModuleBase):
+class Scores:
def assignKNNScore(
self,
field: str,
@@ -28,4 +26,4 @@ class Scores(ModuleBase):
p: int = 2,
**kwargs
) -> saqc.SaQC:
- return self.defer("assignKNNScore", locals())
+ return self._defer("assignKNNScore", locals())
diff --git a/saqc/core/modules/tools.py b/saqc/core/modules/tools.py
index 7667261a722c8762766261f2ae04705cbabfa0ef..bd6f189fe7ff391e5a0215b1de7848464107c5d7 100644
--- a/saqc/core/modules/tools.py
+++ b/saqc/core/modules/tools.py
@@ -7,20 +7,19 @@ from typing_extensions import Literal
import saqc
from saqc.lib.types import FreqString
-from saqc.core.modules.base import ModuleBase
-class Tools(ModuleBase):
- def copy(self, field: str, new_field: str, **kwargs) -> saqc.SaQC:
- return self.defer("copy", locals())
+class Tools:
+ def copyField(self, field: str, new_field: str, **kwargs) -> saqc.SaQC:
+ return self._defer("copyField", locals())
- def drop(self, field: str, **kwargs) -> saqc.SaQC:
- return self.defer("drop", locals())
+ def dropField(self, field: str, **kwargs) -> saqc.SaQC:
+ return self._defer("dropField", locals())
- def rename(self, field: str, new_name: str, **kwargs) -> saqc.SaQC:
- return self.defer("rename", locals())
+ def renameField(self, field: str, new_name: str, **kwargs) -> saqc.SaQC:
+ return self._defer("renameField", locals())
- def mask(
+ def maskTime(
self,
field: str,
mode: Literal["periodic", "mask_field"],
@@ -30,7 +29,7 @@ class Tools(ModuleBase):
closed: bool = True,
**kwargs,
) -> saqc.SaQC:
- return self.defer("mask", locals())
+ return self._defer("maskTime", locals())
def plot(
self,
@@ -44,4 +43,4 @@ class Tools(ModuleBase):
store_kwargs: Optional[dict] = None,
**kwargs,
) -> saqc.SaQC:
- return self.defer("plot", locals())
+ return self._defer("plot", locals())
diff --git a/saqc/core/modules/transformation.py b/saqc/core/modules/transformation.py
index 0032c4e1ae7e36c6358de7493c7a57133e7a41a2..622040d42797c52440aa3ccec96a52dda1126bc9 100644
--- a/saqc/core/modules/transformation.py
+++ b/saqc/core/modules/transformation.py
@@ -6,11 +6,10 @@ from typing import Callable, Optional, Union
import pandas as pd
-from saqc.core.modules.base import ModuleBase
import saqc
-class Transformation(ModuleBase):
+class Transformation:
def transform(
self,
field: str,
@@ -18,4 +17,4 @@ class Transformation(ModuleBase):
freq: Optional[Union[float, str]] = None,
**kwargs
) -> saqc.SaQC:
- return self.defer("transform", locals())
+ return self._defer("transform", locals())
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 0a108c383e2b32cbd5b58646bee5143a13ed0826..2ccb2b318cd7c53ae957fe6c3c1b79b8b650a2c3 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -35,25 +35,21 @@ class CallState:
mask: dios.DictOfSeries
-def processing(module: Optional[str] = None):
+def processing():
# executed on module import
def inner(func):
- func_name = func.__name__
- if module:
- func_name = f"{module}.{func_name}"
-
@wraps(func)
def callWrapper(data, field, flags, *args, **kwargs):
kwargs["to_mask"] = _getMaskingThresh(kwargs)
return func(data, field, flags, *args, **kwargs)
- FUNC_MAP[func_name] = callWrapper
+ FUNC_MAP[func.__name__] = callWrapper
return callWrapper
return inner
-def flagging(masking: MaskingStrT = "all", module: Optional[str] = None):
+def flagging(masking: MaskingStrT = "all"):
# executed on module import
if masking not in ("all", "field", "none"):
@@ -63,8 +59,6 @@ def flagging(masking: MaskingStrT = "all", module: Optional[str] = None):
def inner(func):
func_name = func.__name__
- if module:
- func_name = f"{module}.{func_name}"
# executed if a register-decorated function is called,
# nevertheless if it is called plain or via `SaQC.func`.
@@ -76,7 +70,6 @@ def flagging(masking: MaskingStrT = "all", module: Optional[str] = None):
return _postCall(result, old_state)
FUNC_MAP[func_name] = callWrapper
- callWrapper._module = module
callWrapper._masking = masking
return callWrapper
diff --git a/saqc/funcs/breaks.py b/saqc/funcs/breaks.py
index bc4b034747cb2f525f6016085119ba2f9d3f5f49..e139b886153ddc0e4a52ef7510a4f32f4402b96b 100644
--- a/saqc/funcs/breaks.py
+++ b/saqc/funcs/breaks.py
@@ -28,7 +28,7 @@ from saqc.core.register import _isflagged, flagging
# NOTE:
# masking="none" as we otherwise might interprete
# the masked values as missing data
-@flagging(masking="none", module="breaks")
+@flagging(masking="none")
def flagMissing(
data: DictOfSeries,
field: str,
@@ -68,7 +68,7 @@ def flagMissing(
return data, flags
-@flagging(masking="field", module="breaks")
+@flagging(masking="field")
def flagIsolated(
data: DictOfSeries,
field: str,
@@ -147,7 +147,7 @@ def flagIsolated(
return data, flags
-@flagging(masking="field", module="breaks")
+@flagging(masking="field")
def flagJumps(
data: DictOfSeries,
field: str,
diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py
index 82a304663f128bf587bc7c361892f34cf8b3693a..58af0ad4c1f54e2eb053271c08ed76e933e04a4a 100644
--- a/saqc/funcs/changepoints.py
+++ b/saqc/funcs/changepoints.py
@@ -17,7 +17,7 @@ from saqc.core import flagging, Flags
from saqc.lib.types import FreqString
-@flagging(masking="field", module="changepoints")
+@flagging(masking="field")
def flagChangePoints(
data: DictOfSeries,
field: str,
@@ -112,7 +112,7 @@ def flagChangePoints(
)
-@flagging(masking="field", module="changepoints")
+@flagging(masking="field")
def assignChangePointCluster(
data: DictOfSeries,
field: str,
diff --git a/saqc/funcs/constants.py b/saqc/funcs/constants.py
index e1e671782b0e6886897e1592cb262e90b46240b6..395f0c808dc21e01557d104696e2219d99e9def1 100644
--- a/saqc/funcs/constants.py
+++ b/saqc/funcs/constants.py
@@ -17,7 +17,7 @@ from saqc.lib.tools import customRoller, getFreqDelta, statPass
from saqc.lib.types import FreqString
-@flagging(masking="field", module="constants")
+@flagging(masking="field")
def flagConstants(
data: DictOfSeries,
field: str,
@@ -81,7 +81,7 @@ def flagConstants(
return data, flags
-@flagging(masking="field", module="constants")
+@flagging(masking="field")
def flagByVariance(
data: DictOfSeries,
field: str,
diff --git a/saqc/funcs/curvefit.py b/saqc/funcs/curvefit.py
index b451b7343d73c1bc668654eabee16f278bbc8f69..c29d2bbb4afda3ace36e582c53df0a333baa13b1 100644
--- a/saqc/funcs/curvefit.py
+++ b/saqc/funcs/curvefit.py
@@ -20,7 +20,7 @@ from saqc.lib.ts_operators import (
)
-@flagging(masking="field", module="curvefit")
+@flagging(masking="field")
def fitPolynomial(
data: DictOfSeries,
field: str,
diff --git a/saqc/funcs/drift.py b/saqc/funcs/drift.py
index 5aa171051a66d43e24e6a5e3d81fe15f0197b675..54e658eec8a00703a23ba18f88c84ad5e6abe81d 100644
--- a/saqc/funcs/drift.py
+++ b/saqc/funcs/drift.py
@@ -17,7 +17,7 @@ from saqc.constants import *
from saqc.core.register import flagging
from saqc.core import Flags
from saqc.funcs.changepoints import assignChangePointCluster
-from saqc.funcs.tools import drop, copy
+from saqc.funcs.tools import dropField, copyField
from saqc.lib.tools import detectDeviants
from saqc.lib.types import FreqString, CurveFitter
@@ -27,7 +27,7 @@ LinkageString = Literal[
]
-@flagging(masking="all", module="drift")
+@flagging(masking="all")
def flagDriftFromNorm(
data: DictOfSeries,
field: str,
@@ -147,7 +147,7 @@ def flagDriftFromNorm(
return data, flags
-@flagging(masking="all", module="drift")
+@flagging(masking="all")
def flagDriftFromReference(
data: DictOfSeries,
field: str,
@@ -228,7 +228,7 @@ def flagDriftFromReference(
return data, flags
-@flagging(masking="all", module="drift")
+@flagging(masking="all")
def flagDriftFromScaledNorm(
data: DictOfSeries,
field: str,
@@ -365,7 +365,7 @@ def flagDriftFromScaledNorm(
return data, flags
-@flagging(masking="all", module="drift")
+@flagging(masking="all")
def correctDrift(
data: DictOfSeries,
field: str,
@@ -497,7 +497,7 @@ def correctDrift(
return data, flags
-@flagging(masking="all", module="drift")
+@flagging(masking="all")
def correctRegimeAnomaly(
data: DictOfSeries,
field: str,
@@ -616,7 +616,7 @@ def correctRegimeAnomaly(
return data, flags
-@flagging(masking="all", module="drift")
+@flagging(masking="all")
def correctOffset(
data: DictOfSeries,
field: str,
@@ -661,7 +661,7 @@ def correctOffset(
flags : saqc.Flags
The quality flags of data
"""
- data, flags = copy(data, field, flags, field + "_CPcluster")
+ data, flags = copyField(data, field, flags, field + "_CPcluster")
data, flags = assignChangePointCluster(
data,
field + "_CPcluster",
@@ -680,7 +680,7 @@ def correctOffset(
lambda x, p1: np.array([p1] * x.shape[0]),
tolerance=tolerance,
)
- data, flags = drop(data, field + "_CPcluster", flags)
+ data, flags = dropField(data, field + "_CPcluster", flags)
return data, flags
@@ -714,7 +714,7 @@ def _driftFit(x, shift_target, cal_mean, driftModel):
return data_fit, data_shift
-@flagging(masking="all", module="drift")
+@flagging(masking="all")
def flagRegimeAnomaly(
data: DictOfSeries,
field: str,
@@ -791,7 +791,7 @@ def flagRegimeAnomaly(
)
-@flagging(masking="all", module="drift")
+@flagging(masking="all")
def assignRegimeAnomaly(
data: DictOfSeries,
field: str,
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index 8fa9417da75c5fbd2bcae88662fa309bc2f4e0b0..30924b46bda36e2e9ef24d7e4b8599caecf892d9 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -10,7 +10,7 @@ from saqc.core import flagging, processing, Flags
import warnings
-@flagging(masking="field", module="flagtools")
+@flagging(masking="field")
def forceFlags(
data: DictOfSeries, field: str, flags: Flags, flag: float = BAD, **kwargs
) -> Tuple[DictOfSeries, Flags]:
@@ -45,7 +45,7 @@ def forceFlags(
# masking='none' is sufficient because call is redirected
-@flagging(masking="none", module="flagtools")
+@flagging(masking="none")
def clearFlags(
data: DictOfSeries, field: str, flags: Flags, **kwargs
) -> Tuple[DictOfSeries, Flags]:
@@ -90,7 +90,7 @@ def clearFlags(
return forceFlags(data, field, flags, flag=UNFLAGGED, **kwargs)
-@flagging(masking="none", module="flagtools")
+@flagging(masking="none")
def flagUnflagged(
data: DictOfSeries, field: str, flags: Flags, flag: float = BAD, **kwargs
) -> Tuple[DictOfSeries, Flags]:
@@ -132,7 +132,7 @@ def flagUnflagged(
return data, flags
-@flagging(masking="field", module="flagtools")
+@flagging(masking="field")
def flagManual(
data: DictOfSeries,
field: str,
@@ -276,7 +276,7 @@ def flagManual(
return data, flags
-@flagging(module="flagtools")
+@flagging()
def flagDummy(
data: DictOfSeries, field: str, flags: Flags, **kwargs
) -> Tuple[DictOfSeries, Flags]:
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index c04ca59ceb995fe5bb8485edc9f048ccd7cf674a..99e460bc7d95b3af4146f8cc30e5f061d1774907 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -72,7 +72,6 @@ def _execGeneric(
globs = {
"isflagged": partial(_dslIsFlagged, flags),
"ismissing": lambda var: pd.isnull(var),
- "mask": lambda cond: data[cond.name].mask(cond),
"this": field,
"GOOD": GOOD,
"BAD": BAD,
@@ -83,8 +82,8 @@ def _execGeneric(
return func(*args)
-@processing(module="generic")
-def process(
+@processing()
+def genericProcess(
data: DictOfSeries,
field: str,
flags: Flags,
@@ -151,8 +150,8 @@ def process(
return data, flags
-@flagging(masking="all", module="generic")
-def flag(
+@flagging(masking="all")
+def genericFlag(
data: DictOfSeries,
field: str,
flags: Flags,
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index f5073db7408d642617ff6142fed3238b0d2cb64e..53e4f1e6e341c896d65a2dd6bc482fd5052dc237 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -30,7 +30,7 @@ _SUPPORTED_METHODS = Literal[
]
-@flagging(masking="field", module="interpolation")
+@flagging(masking="field")
def interpolateByRolling(
data: DictOfSeries,
field: str,
@@ -105,7 +105,7 @@ def interpolateByRolling(
return data, flags
-@flagging(masking="field", module="interpolation")
+@flagging(masking="field")
def interpolateInvalid(
data: DictOfSeries,
field: str,
@@ -189,7 +189,7 @@ def _resampleOverlapping(data: pd.Series, freq: str, fill_value):
return data.fillna(fill_value).astype(dtype)
-@processing(module="interpolation")
+@processing()
def interpolateIndex(
data: DictOfSeries,
field: str,
diff --git a/saqc/funcs/noise.py b/saqc/funcs/noise.py
index b47d4c03f2ff6b03a99f026b5847eb24c474cc25..fb70835d98f8e64b56e1ffe4e0a0839159f81c36 100644
--- a/saqc/funcs/noise.py
+++ b/saqc/funcs/noise.py
@@ -11,7 +11,7 @@ from saqc.lib.types import FreqString
from saqc.lib.tools import statPass
-@flagging(masking="field", module="noise")
+@flagging(masking="field")
def flagByStatLowPass(
data: DictOfSeries,
field: str,
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index 5edf327cb731032e6329f2a7171c4730478d93dc..bc7325eed65553c3491ea78051d56496ed19e788 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -23,7 +23,7 @@ from saqc.funcs.transformation import transform
import saqc.lib.ts_operators as ts_ops
-@flagging(masking="field", module="outliers")
+@flagging(masking="field")
def flagByStray(
data: DictOfSeries,
field: str,
@@ -399,7 +399,7 @@ def _expFit(
return val_frame.index[sorted_i[iter_index:]]
-@flagging(masking="all", module="outliers")
+@flagging(masking="all")
def flagMVScores(
data: DictOfSeries,
field: str,
@@ -604,7 +604,7 @@ def flagMVScores(
return data, flags
-@flagging(masking="field", module="outliers")
+@flagging(masking="field")
def flagRaise(
data: DictOfSeries,
field: str,
@@ -792,7 +792,7 @@ def flagRaise(
return data, flags
-@flagging(masking="field", module="outliers")
+@flagging(masking="field")
def flagMAD(
data: DictOfSeries,
field: str,
@@ -862,7 +862,7 @@ def flagMAD(
return data, flags
-@flagging(masking="field", module="outliers")
+@flagging(masking="field")
def flagOffset(
data: DictOfSeries,
field: str,
@@ -1030,7 +1030,7 @@ def flagOffset(
return data, flags
-@flagging(masking="field", module="outliers")
+@flagging(masking="field")
def flagByGrubbs(
data: DictOfSeries,
field: str,
@@ -1153,7 +1153,7 @@ def flagByGrubbs(
return data, flags
-@flagging(masking="field", module="outliers")
+@flagging(masking="field")
def flagRange(
data: DictOfSeries,
field: str,
@@ -1196,7 +1196,7 @@ def flagRange(
return data, flags
-@flagging(masking="all", module="outliers")
+@flagging(masking="all")
def flagCrossStatistic(
data: DictOfSeries,
field: str,
diff --git a/saqc/funcs/pattern.py b/saqc/funcs/pattern.py
index de684402abe35cf4467f52a56f7261c21236af5b..a5c23e0323dcebc4c88cc4c735b7d20c7d292b49 100644
--- a/saqc/funcs/pattern.py
+++ b/saqc/funcs/pattern.py
@@ -12,7 +12,7 @@ from saqc.core.register import flagging
from saqc.lib.tools import customRoller
-@flagging(masking="field", module="pattern")
+@flagging(masking="field")
def flagPatternByWavelet(
data,
field,
@@ -180,7 +180,7 @@ def calculateDistanceByDTW(
return distances.reindex(index=data.index) # reinsert NaNs
-@flagging(masking="field", module="pattern")
+@flagging(masking="field")
def flagPatternByDTW(
data,
field,
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 57fee06b197f183f57b63020b72f3431cf4c85a7..d617798384dac1b6bef059c6e59c3d0d3cd47871 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -27,7 +27,7 @@ METHOD2ARGS = {
}
-@processing(module="resampling")
+@processing()
def linear(
data: DictOfSeries, field: str, flags: Flags, freq: str, **kwargs
) -> Tuple[DictOfSeries, Flags]:
@@ -70,7 +70,7 @@ def linear(
return interpolateIndex(data, field, flags, freq, "time", **kwargs)
-@processing(module="resampling")
+@processing()
def interpolate(
data: DictOfSeries,
field: str,
@@ -135,7 +135,7 @@ def interpolate(
)
-@processing(module="resampling")
+@processing()
def shift(
data: DictOfSeries,
field: str,
@@ -216,7 +216,7 @@ def shift(
return data, flags
-@processing(module="resampling")
+@processing()
def resample(
data: DictOfSeries,
field: str,
@@ -412,7 +412,7 @@ def _inverseShift(
return source.fillna(fill_value).astype(dtype, copy=False)
-@processing(module="resampling")
+@processing()
def reindexFlags(
data: DictOfSeries,
field: str,
@@ -542,6 +542,6 @@ def reindexFlags(
flags.history[field] = flags.history[field].append(history)
if drop:
- data, flags = tools.drop(data=data, flags=flags, field=source)
+ data, flags = tools.dropField(data=data, flags=flags, field=source)
return data, flags
diff --git a/saqc/funcs/residues.py b/saqc/funcs/residues.py
index 6b62a75c32b1ac0b5f216bf2bbf5f799f3b04710..bc21514671c6a6bb3bbf9c57aace11b484ac5c91 100644
--- a/saqc/funcs/residues.py
+++ b/saqc/funcs/residues.py
@@ -12,7 +12,7 @@ from saqc.funcs.rolling import roll
from saqc.funcs.curvefit import fitPolynomial
-@flagging(masking="field", module="residues")
+@flagging(masking="field")
def calculatePolynomialResidues(
data: DictOfSeries,
field: str,
@@ -109,7 +109,7 @@ def calculatePolynomialResidues(
)
-@flagging(masking="field", module="residues")
+@flagging(masking="field")
def calculateRollingResidues(
data: DictOfSeries,
field: str,
diff --git a/saqc/funcs/rolling.py b/saqc/funcs/rolling.py
index 79da68bc7d86ce375367584d748543c806ba6a58..5eb13520115314e2735234c1aeb2feccee23bb92 100644
--- a/saqc/funcs/rolling.py
+++ b/saqc/funcs/rolling.py
@@ -11,7 +11,7 @@ from saqc.core import flagging, Flags
from saqc.lib.tools import getFreqDelta
-@flagging(masking="field", module="rolling")
+@flagging(masking="field")
def roll(
data: DictOfSeries,
field: str,
diff --git a/saqc/funcs/scores.py b/saqc/funcs/scores.py
index a86e0206374251bd2c50c3a796fdf04b7c95a199..dba04b3302cad157a5d489f7fdeb9212c503c9d2 100644
--- a/saqc/funcs/scores.py
+++ b/saqc/funcs/scores.py
@@ -12,7 +12,7 @@ from saqc.lib.tools import toSequence
import saqc.lib.ts_operators as ts_ops
-@flagging(masking="all", module="scores")
+@flagging(masking="all")
def assignKNNScore(
data: DictOfSeries,
field: str,
diff --git a/saqc/funcs/tools.py b/saqc/funcs/tools.py
index 67494c275792c24e1e4b329fbb4d96f313c61c6b..c8cace05fc624445cd2a2e61f410780eefc5b2fd 100644
--- a/saqc/funcs/tools.py
+++ b/saqc/funcs/tools.py
@@ -20,8 +20,8 @@ from saqc.lib.plotting import makeFig
_MPL_DEFAULT_BACKEND = mpl.get_backend()
-@processing(module="tools")
-def copy(
+@processing()
+def copyField(
data: DictOfSeries, field: str, flags: Flags, new_field: str, **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
@@ -57,8 +57,8 @@ def copy(
return data, flags
-@processing(module="tools")
-def drop(
+@processing()
+def dropField(
data: DictOfSeries, field: str, flags: Flags, **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
@@ -87,8 +87,8 @@ def drop(
return data, flags
-@processing(module="tools")
-def rename(
+@processing()
+def renameField(
data: DictOfSeries, field: str, flags: Flags, new_name: str, **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
@@ -119,8 +119,8 @@ def rename(
return data, flags
-@processing(module="tools")
-def mask(
+@processing()
+def maskTime(
data: DictOfSeries,
field: str,
flags: Flags,
@@ -244,7 +244,7 @@ def mask(
return data, flags
-@processing(module="tools")
+@processing()
def plot(
data: DictOfSeries,
field: str,
diff --git a/saqc/funcs/transformation.py b/saqc/funcs/transformation.py
index 0fa9eb8f0d7164f321c3a3f1a723ac36e5179513..357e948cdbbf47a945b2a8f1e16ce5e1fa064197 100644
--- a/saqc/funcs/transformation.py
+++ b/saqc/funcs/transformation.py
@@ -9,7 +9,7 @@ from dios import DictOfSeries
from saqc.core import flagging, Flags
-@flagging(masking="field", module="transformation")
+@flagging(masking="field")
def transform(
data: DictOfSeries,
field: str,
diff --git a/sphinx-doc/cook_books_md/OutlierDetection.md b/sphinx-doc/cook_books_md/OutlierDetection.md
index 215d68565b5391a17de7ac316310cbdcf22c8fa9..e521eb948cd4f9eab38dfe59498d8e89308e655a 100644
--- a/sphinx-doc/cook_books_md/OutlierDetection.md
+++ b/sphinx-doc/cook_books_md/OutlierDetection.md
@@ -122,13 +122,13 @@ result_data.plot()
## Residues calculation
We want to evaluate the residues of the model, in order to score the outlierish-nes of every point.
-First, we retrieve the residues via the :py:func:`saqc.generic.process <docs.func_modules.generic.process>` method.
+First, we retrieve the residues via the :py:func:`saqc.genericProcess <docs.func_modules.genericProcess>` method.
The method generates a new variable, resulting from the processing of other variables. It automatically
generates the field name it gets passed - so we do not have to generate new variable beforehand. The function we apply
is just the computation of the variables difference for any timestep.
```python
-i_saqc = i_saqc.generic.process('incidents_residues', func=lambda incidents, incidents_model:incidents - incidents_model)
+i_saqc = i_saqc.genericProcess('incidents_residues', func=lambda incidents, incidents_model:incidents - incidents_model)
```
Next, we score the residues simply by computing their [Z-scores](https://en.wikipedia.org/wiki/Standard_score).
@@ -140,7 +140,7 @@ i_saqc = i_saqc.rolling.roll(field='incidents_residues', target='residues_mean',
i_saqc = i_saqc.rolling.roll(field='incidents_residues', target='residues_std',
window='27D',
func=np.std)
-i_saqc = i_saqc.generic.process(field='incidents_scores',
+i_saqc = i_saqc.genericProcess(field='incidents_scores',
func=lambda This, residues_mean, residues_std: (
This - residues_mean) / residues_std)
```
diff --git a/tests/core/test_core.py b/tests/core/test_core.py
index 095f1a52453b0349b23df4f8a8e1f8f4f565c63a..e64f6b9a5f0317d136f16e9328b2e921f339227d 100644
--- a/tests/core/test_core.py
+++ b/tests/core/test_core.py
@@ -45,7 +45,7 @@ def test_duplicatedVariable():
data = initData(1)
var1 = data.columns[0]
- pdata, pflags = SaQC(data).flagtools.flagDummy(var1).getResult()
+ pdata, pflags = SaQC(data).flagDummy(var1).getResult()
if isinstance(pflags.columns, pd.MultiIndex):
cols = pflags.columns.get_level_values(0).drop_duplicates()
diff --git a/tests/core/test_reader.py b/tests/core/test_reader.py
index 807fe487b125e8f40047ca183c7659b071fbf0f4..e13f2976f97876e957c8640ac4e5680543b857e7 100644
--- a/tests/core/test_reader.py
+++ b/tests/core/test_reader.py
@@ -47,7 +47,7 @@ def test_variableRegex(data):
]
for regex, expected in tests:
- fobj = writeIO(header + "\n" + f"{regex} ; flagtools.flagDummy()")
+ fobj = writeIO(header + "\n" + f"{regex} ; flagDummy()")
saqc = fromConfig(fobj, data=data)
result = [field for field, _ in saqc.called]
assert np.all(result == expected)
@@ -59,25 +59,25 @@ def test_inlineComments(data):
"""
config = f"""
varname ; test
- pre2 ; flagtools.flagDummy() # test
+ pre2 ; flagDummy() # test
"""
saqc = fromConfig(writeIO(config), data)
_, func = saqc.called[0]
- assert func[0] == FUNC_MAP["flagtools.flagDummy"]
+ assert func[0] == FUNC_MAP["flagDummy"]
def test_configReaderLineNumbers(data):
config = f"""
varname ; test
- #temp1 ; flagtools.flagDummy()
- pre1 ; flagtools.flagDummy()
- pre2 ; flagtools.flagDummy()
- SM ; flagtools.flagDummy()
- #SM ; flagtools.flagDummy()
- # SM1 ; flagtools.flagDummy()
-
- SM1 ; flagtools.flagDummy()
+ #temp1 ; flagDummy()
+ pre1 ; flagDummy()
+ pre2 ; flagDummy()
+ SM ; flagDummy()
+ #SM ; flagDummy()
+ # SM1 ; flagDummy()
+
+ SM1 ; flagDummy()
"""
planned = readFile(writeIO(config))
expected = [4, 5, 6, 10]
@@ -91,14 +91,14 @@ def test_configFile(data):
config = f"""
varname ; test
- #temp1 ; flagtools.flagDummy()
- pre1; flagtools.flagDummy()
- pre2 ;flagtools.flagDummy()
- SM ; flagtools.flagDummy()
- #SM ; flagtools.flagDummy()
- # SM1 ; flagtools.flagDummy()
+ #temp1 ; flagDummy()
+ pre1; flagDummy()
+ pre2 ;flagDummy()
+ SM ; flagDummy()
+ #SM ; flagDummy()
+ # SM1 ; flagDummy()
- SM1;flagtools.flagDummy()
+ SM1;flagDummy()
"""
fromConfig(writeIO(config), data)
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index 13d85a554ace6df9cfe1eb31e7486138fcdb9374..3e06597e5a676dbc567e6a20470f6cda56968ad3 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -155,9 +155,7 @@ def test_positionalTranslatorIntegration():
translator = PositionalTranslator()
saqc = SaQC(data=data, scheme=translator)
- saqc = saqc.breaks.flagMissing(col).outliers.flagRange(
- col, min=3, max=10, flag=DOUBTFUL
- )
+ saqc = saqc.flagMissing(col).flagRange(col, min=3, max=10, flag=DOUBTFUL)
data, flags = saqc.getResult()
for field in flags.columns:
@@ -177,7 +175,7 @@ def test_dmpTranslatorIntegration():
translator = DmpTranslator()
saqc = SaQC(data=data, scheme=translator)
- saqc = saqc.breaks.flagMissing(col).outliers.flagRange(col, min=3, max=10)
+ saqc = saqc.flagMissing(col).flagRange(col, min=3, max=10)
data, flags = saqc.getResult()
qflags = flags.xs("quality_flag", axis="columns", level=1)
@@ -187,7 +185,7 @@ def test_dmpTranslatorIntegration():
qcause = flags.xs("quality_cause", axis="columns", level=1)
assert qflags.isin(translator._forward.keys()).all(axis=None)
- assert qfunc.isin({"", "breaks.flagMissing", "outliers.flagRange"}).all(axis=None)
+ assert qfunc.isin({"", "flagMissing", "flagRange"}).all(axis=None)
assert (qcause[qflags[col] == "BAD"] == "OTHER").all(axis=None)
round_trip = translator.backward(translator.forward(flags))
@@ -211,12 +209,10 @@ def test_dmpValidCombinations():
saqc = SaQC(data=data, scheme=translator)
with pytest.raises(ValueError):
- saqc.outliers.flagRange(
- col, min=3, max=10, cause="SOMETHING_STUPID"
- ).getResult()
+ saqc.flagRange(col, min=3, max=10, cause="SOMETHING_STUPID").getResult()
with pytest.raises(ValueError):
- saqc.outliers.flagRange(col, min=3, max=10, cause="").getResult()
+ saqc.flagRange(col, min=3, max=10, cause="").getResult()
def _buildupSaQCObjects():
@@ -233,9 +229,9 @@ def _buildupSaQCObjects():
out = []
for _ in range(2):
saqc = SaQC(data=data, flags=flags)
- saqc = saqc.outliers.flagRange(
- field=col, min=5, max=6, to_mask=False
- ).outliers.flagRange(col, min=3, max=10, to_mask=False)
+ saqc = saqc.flagRange(field=col, min=5, max=6, to_mask=False).flagRange(
+ col, min=3, max=10, to_mask=False
+ )
flags = saqc._flags
out.append(saqc)
return out
diff --git a/tests/funcs/test_functions.py b/tests/funcs/test_functions.py
index 5b8c926fdd835a48e2a4fd25f130ce8de95aa77d..7b8885d7434998f7e6b37e87a05f43d3f80ec129 100644
--- a/tests/funcs/test_functions.py
+++ b/tests/funcs/test_functions.py
@@ -15,7 +15,7 @@ from saqc.funcs.drift import (
)
from saqc.funcs.outliers import flagCrossStatistic, flagRange
from saqc.funcs.flagtools import flagManual, forceFlags, clearFlags
-from saqc.funcs.tools import drop, copy, mask
+from saqc.funcs.tools import dropField, copyField, maskTime
from saqc.funcs.resampling import reindexFlags
from saqc.funcs.breaks import flagIsolated
@@ -93,8 +93,8 @@ def test_flagSesonalRange(data, field):
start = f"{test['startmonth']:02}-{test['startday']:02}T00:00:00"
end = f"{test['endmonth']:02}-{test['endday']:02}T00:00:00"
- data, flags = copy(data, field, flags, field + "_masked")
- data, flags = mask(
+ data, flags = copyField(data, field, flags, field + "_masked")
+ data, flags = maskTime(
data,
newfield,
flags,
@@ -110,7 +110,7 @@ def test_flagSesonalRange(data, field):
data, flags = reindexFlags(
data, field, flags, method="match", source=newfield, flag=BAD
)
- data, flags = drop(data, newfield, flags)
+ data, flags = dropField(data, newfield, flags)
flagged = flags[field] > UNFLAGGED
assert flagged.sum() == expected
diff --git a/tests/funcs/test_generic_api_functions.py b/tests/funcs/test_generic_api_functions.py
index f5c10b90cb8b9fed61ae0acd6b77f30f74f094e3..360418cbb4254c206784746577a4a23550b218db 100644
--- a/tests/funcs/test_generic_api_functions.py
+++ b/tests/funcs/test_generic_api_functions.py
@@ -6,7 +6,7 @@ import pandas as pd
from saqc.constants import *
from saqc.core.register import flagging
-from saqc.funcs.tools import mask
+from saqc.funcs.tools import maskTime
from saqc import SaQC
from tests.common import initData, flagAll
@@ -24,7 +24,7 @@ def test_addFieldFlagGeneric(data):
saqc = SaQC(data=data)
func = lambda var1: pd.Series(False, index=data[var1.name].index)
- data, flags = saqc.generic.flag("tmp1", func, flag=BAD).getResult()
+ data, flags = saqc.genericFlag("tmp1", func, flag=BAD).getResult()
assert "tmp1" in flags.columns and "tmp1" not in data
@@ -32,32 +32,11 @@ def test_addFieldProcGeneric(data):
saqc = SaQC(data=data)
func = lambda: pd.Series([])
- data, flags = saqc.generic.process("tmp1", func, flag=BAD).getResult(raw=True)
+ data, flags = saqc.genericProcess("tmp1", func, flag=BAD).getResult(raw=True)
assert "tmp1" in data.columns and data["tmp1"].empty
func = lambda var1, var2: var1 + var2
- data, flags = saqc.generic.process("tmp2", func, flag=BAD).getResult()
+ data, flags = saqc.genericProcess("tmp2", func, flag=BAD).getResult()
assert "tmp2" in data.columns and (data["tmp2"] == data["var1"] + data["var2"]).all(
axis=None
)
-
-
-def test_mask(data):
- saqc = SaQC(data=data)
- data_org = data.copy(deep=True)
- mean = data["var1"] / 2
-
- data, _ = saqc.generic.process(
- "var1", lambda var1: mask(var1 < mean), flag=BAD
- ).getResult()
- assert (
- (data["var1"].isna()) == (data_org["var1"] < 10) & data_org["var1"].isna()
- ).all(axis=None)
-
- data, flags = saqc.generic.process(
- "tmp", lambda var1: mask(var1 < mean), flag=BAD
- ).getResult()
- assert ("tmp" in data.columns) and ("tmp" in flags.columns)
- assert (
- (data["tmp"].isna()) == (data_org["var1"] < 10) & data_org["var1"].isna()
- ).all(axis=None)
diff --git a/tests/funcs/test_generic_config_functions.py b/tests/funcs/test_generic_config_functions.py
index 1298e7317d49dd01e6f4e0bb8a8b4b2b0587f697..2046ff8e9b02114e1390eb658323068a78ebcba8 100644
--- a/tests/funcs/test_generic_config_functions.py
+++ b/tests/funcs/test_generic_config_functions.py
@@ -55,7 +55,7 @@ def test_missingIdentifier(data):
]
for test in tests:
- func = _compileGeneric(f"generic.flag(func={test})", flags)
+ func = _compileGeneric(f"genericFlag(func={test})", flags)
with pytest.raises(NameError):
_execGeneric(flags, data, func, field="")
@@ -85,7 +85,7 @@ def test_typeError():
for test in tests:
with pytest.raises(TypeError):
- _compileGeneric(f"generic.flag(func={test})", flags)
+ _compileGeneric(f"genericFlag(func={test})", flags)
def test_comparisonOperators(data):
@@ -103,7 +103,7 @@ def test_comparisonOperators(data):
]
for test, expected in tests:
- func = _compileGeneric(f"generic.flag(func={test})", flags)
+ func = _compileGeneric(f"genericFlag(func={test})", flags)
result = _execGeneric(flags, data, func, field=var1)
assert np.all(result == expected)
@@ -123,7 +123,7 @@ def test_arithmeticOperators(data):
]
for test, expected in tests:
- func = _compileGeneric(f"generic.process(func={test})", flags)
+ func = _compileGeneric(f"genericProcess(func={test})", flags)
result = _execGeneric(flags, data, func, field=var1)
assert np.all(result == expected)
@@ -132,20 +132,15 @@ def test_nonReduncingBuiltins(data):
flags = initFlagsLike(data)
var1, *_ = data.columns
this = var1
- mean = data[var1].mean()
tests = [
(f"abs({this})", np.abs(data[this])),
(f"log({this})", np.log(data[this])),
(f"exp({this})", np.exp(data[this])),
- (
- f"ismissing(mask({this} < {mean}))",
- data[this].mask(data[this] < mean).isna(),
- ),
]
for test, expected in tests:
- func = _compileGeneric(f"generic.process(func={test})", flags)
+ func = _compileGeneric(f"genericProcess(func={test})", flags)
result = _execGeneric(flags, data, func, field=this)
assert (result == expected).all()
@@ -166,7 +161,7 @@ def test_reduncingBuiltins(data):
]
for test, expected in tests:
- func = _compileGeneric(f"generic.process(func={test})", flags)
+ func = _compileGeneric(f"genericProcess(func={test})", flags)
result = _execGeneric(flags, data, func, field=this.name)
assert result == expected
@@ -184,7 +179,7 @@ def test_ismissing(data):
]
for test, expected in tests:
- func = _compileGeneric(f"generic.flag(func={test})", flags)
+ func = _compileGeneric(f"genericFlag(func={test})", flags)
result = _execGeneric(flags, data, func, this.name)
assert np.all(result == expected)
@@ -202,7 +197,7 @@ def test_bitOps(data):
]
for test, expected in tests:
- func = _compileGeneric(f"generic.flag(func={test})", flags)
+ func = _compileGeneric(f"genericFlag(func={test})", flags)
result = _execGeneric(flags, data, func, this)
assert np.all(result == expected)
@@ -226,7 +221,7 @@ def test_isflagged(data):
for i, (test, expected) in enumerate(tests):
try:
- func = _compileGeneric(f"generic.flag(func={test}, flag=BAD)", flags)
+ func = _compileGeneric(f"genericFlag(func={test}, flag=BAD)", flags)
result = _execGeneric(flags, data, func, field=None)
assert np.all(result == expected)
except Exception:
@@ -237,7 +232,7 @@ def test_isflagged(data):
for comp in [">", ">=", "==", "!=", "<", "<="]:
fails = f"isflagged({var1}, comparator='{comp}')"
- func = _compileGeneric(f"generic.flag(func={fails}, flag=BAD)", flags)
+ func = _compileGeneric(f"genericFlag(func={fails}, flag=BAD)", flags)
with pytest.raises(ValueError):
_execGeneric(flags, data, func, field=None)
@@ -247,8 +242,8 @@ def test_variableAssignments(data):
config = f"""
varname ; test
- dummy1 ; generic.process(func=var1 + var2)
- dummy2 ; generic.flag(func=var1 + var2 > 0)
+ dummy1 ; genericProcess(func=var1 + var2)
+ dummy2 ; genericFlag(func=var1 + var2 > 0)
"""
fobj = writeIO(config)
@@ -264,8 +259,8 @@ def test_variableAssignments(data):
def test_processMultiple(data_diff):
config = f"""
varname ; test
- dummy ; generic.process(func=var1 + 1)
- dummy ; generic.process(func=var2 - 1)
+ dummy ; genericProcess(func=var1 + 1)
+ dummy ; genericProcess(func=var2 - 1)
"""
fobj = writeIO(config)
diff --git a/tests/funcs/test_harm_funcs.py b/tests/funcs/test_harm_funcs.py
index 16cac1df19ff9128963b988603dadaafed889b23..d645e1bd9478321921afc4ab58ba7660510137f4 100644
--- a/tests/funcs/test_harm_funcs.py
+++ b/tests/funcs/test_harm_funcs.py
@@ -9,7 +9,7 @@ import dios
from saqc.core import initFlagsLike, Flags
from saqc.constants import BAD, UNFLAGGED
from saqc.funcs.resampling import linear, interpolate, shift, reindexFlags, resample
-from saqc.funcs.tools import copy, drop
+from saqc.funcs.tools import copyField, dropField
from tests.common import checkDataFlagsInvariants
@@ -135,7 +135,7 @@ def test_harmSingleVarIntermediateFlagging(data, reshaper):
pre_data = data.copy()
pre_flags = flags.copy()
- data, flags = copy(data, field, flags, field + "_interpolated")
+ data, flags = copyField(data, field, flags, field + "_interpolated")
data, flags = linear(data, field + "_interpolated", flags, freq=freq)
checkDataFlagsInvariants(data, flags, field + "_interpolated", identical=True)
assert data[field + "_interpolated"].index.inferred_freq == freq
@@ -145,7 +145,7 @@ def test_harmSingleVarIntermediateFlagging(data, reshaper):
data, flags = reindexFlags(
data, field, flags, method="inverse_" + reshaper, source=field + "_interpolated"
)
- data, flags = drop(data, field + "_interpolated", flags)
+ data, flags = dropField(data, field + "_interpolated", flags)
assert len(data[field]) == len(flags[field])
assert data[field].equals(pre_data[field])
@@ -235,7 +235,7 @@ def test_harmSingleVarInterpolationAgg(data, params, expected):
pre_flaggger = flags.copy()
method, freq = params
- data_harm, flags_harm = copy(data, "data", flags, "data_harm")
+ data_harm, flags_harm = copyField(data, "data", flags, "data_harm")
data_harm, flags_harm = resample(
data_harm, h_field, flags_harm, freq, func=np.sum, method=method
)
@@ -246,7 +246,7 @@ def test_harmSingleVarInterpolationAgg(data, params, expected):
data_deharm, flags_deharm = reindexFlags(
data_harm, field, flags_harm, source=h_field, method="inverse_" + method
)
- data_deharm, flags_deharm = drop(data_deharm, h_field, flags_deharm)
+ data_deharm, flags_deharm = dropField(data_deharm, h_field, flags_deharm)
checkDataFlagsInvariants(data_deharm, flags_deharm, field, identical=True)
assert data_deharm[field].equals(pre_data[field])
assert flags_deharm[field].equals(pre_flaggger[field])
@@ -319,7 +319,7 @@ def test_harmSingleVarInterpolationShift(data, params, expected):
pre_flags = flags.copy()
method, freq = params
- data_harm, flags_harm = copy(data, "data", flags, "data_harm")
+ data_harm, flags_harm = copyField(data, "data", flags, "data_harm")
data_harm, flags_harm = shift(data_harm, h_field, flags_harm, freq, method=method)
assert data_harm[h_field].equals(expected)
checkDataFlagsInvariants(data_harm, flags_harm, field, identical=True)
@@ -329,6 +329,6 @@ def test_harmSingleVarInterpolationShift(data, params, expected):
)
checkDataFlagsInvariants(data_deharm, flags_deharm, field, identical=True)
- data_deharm, flags_deharm = drop(data_deharm, h_field, flags_deharm)
+ data_deharm, flags_deharm = dropField(data_deharm, h_field, flags_deharm)
assert data_deharm[field].equals(pre_data[field])
assert flags_deharm[field].equals(pre_flags[field])
diff --git a/tests/funcs/test_modelling.py b/tests/funcs/test_modelling.py
index a37b013b567847b3b456320796392c7be378ec90..ff244da97d922915fba48ff905152809825b5b7c 100644
--- a/tests/funcs/test_modelling.py
+++ b/tests/funcs/test_modelling.py
@@ -8,7 +8,7 @@ import dios
from saqc import BAD, UNFLAGGED
from saqc.core import initFlagsLike
-from saqc.funcs.tools import mask
+from saqc.funcs.tools import maskTime
from saqc.funcs.residues import calculatePolynomialResidues, calculateRollingResidues
from tests.fixtures import *
@@ -81,7 +81,7 @@ def test_modelling_mask(dat):
flags[:, field] = BAD
common = dict(data=data, field=field, flags=flags, mode="periodic")
- data_seasonal, flags_seasonal = mask(
+ data_seasonal, flags_seasonal = maskTime(
**common, start="20:00", end="40:00", closed=False
)
flagscol = flags_seasonal[field]
@@ -89,13 +89,13 @@ def test_modelling_mask(dat):
assert all(flags_seasonal[field][m] == UNFLAGGED)
assert all(data_seasonal[field][m].isna())
- data_seasonal, flags_seasonal = mask(**common, start="15:00:00", end="02:00:00")
+ data_seasonal, flags_seasonal = maskTime(**common, start="15:00:00", end="02:00:00")
flagscol = flags_seasonal[field]
m = (15 <= flagscol.index.hour) & (flagscol.index.hour <= 2)
assert all(flags_seasonal[field][m] == UNFLAGGED)
assert all(data_seasonal[field][m].isna())
- data_seasonal, flags_seasonal = mask(
+ data_seasonal, flags_seasonal = maskTime(
**common, start="03T00:00:00", end="10T00:00:00"
)
flagscol = flags_seasonal[field]
@@ -107,7 +107,7 @@ def test_modelling_mask(dat):
mask_ser[::5] = True
data["mask_ser"] = mask_ser
flags = initFlagsLike(data)
- data_masked, flags_masked = mask(
+ data_masked, flags_masked = maskTime(
data, "data", flags, mode="mask_field", mask_field="mask_ser"
)
m = mask_ser
diff --git a/tests/funcs/test_tools.py b/tests/funcs/test_tools.py
index 179b611440340bccde1d63f6461009cde9fb7f92..c181277cc01129b2921ef6e494014a8ceeacb748 100644
--- a/tests/funcs/test_tools.py
+++ b/tests/funcs/test_tools.py
@@ -18,21 +18,19 @@ def test_makeFig():
)
d_saqc = saqc.SaQC(data)
d_saqc = (
- d_saqc.outliers.flagRange("data", max=500)
- .outliers.flagRange("data", max=400)
- .outliers.flagRange("data", max=300)
+ d_saqc.flagRange("data", max=500)
+ .flagRange("data", max=400)
+ .flagRange("data", max=300)
)
# not interactive, no storing
dummy_path = ""
- d_saqc = d_saqc.tools.plot(field="data", path="")
- d_saqc = d_saqc.tools.plot(
+ d_saqc = d_saqc.plot(field="data", path="")
+ d_saqc = d_saqc.plot(
field="data", path=dummy_path, plot_kwargs={"history": "valid"}, stats=True
)
- d_saqc = d_saqc.tools.plot(
- field="data", path=dummy_path, plot_kwargs={"history": "all"}
- )
- d_saqc = d_saqc.tools.plot(
+ d_saqc = d_saqc.plot(field="data", path=dummy_path, plot_kwargs={"history": "all"})
+ d_saqc = d_saqc.plot(
field="data", path=dummy_path, plot_kwargs={"slice": "2000-10"}, stats=True
)