CHANGELOG.md

@@ -13,11 +13,16 @@ This changelog starts with version 2.0.0. Basically all parts of the system, inc
 [List of commits](https://git.ufz.de/rdm-software/saqc/-/compare/v2.0.1...develop)
 
 ### Added
+- generic documentation module `docurator.py` added to `lib`
 ### Changed
+- documentation pipeline changed to base on methods decorators
 - `flagOffsets` parameters `thresh` and `thresh_relative` now both are optional
+- corrected false notion of *residual* concept (old notion: *residue* got replaced by *residual*)
 ### Removed
 ### Fixed
 - `flagOffset` bug with zero-valued threshold
+- `flagCrossStatistics` bug with unaligned input variables
+- `plot` fixed data loss when using *dfilter* kwarg
 
 ## [2.0.1](https://git.ufz.de/rdm-software/saqc/-/tags/v2.0.1) - 2021-12-20
 [List of commits](https://git.ufz.de/rdm-software/saqc/-/compare/v2.0.0...v2.0.1)

requirements.txt

@@ -6,8 +6,8 @@ Click==8.0.3
 dtw==1.4.0
 hypothesis==6.34.1
 matplotlib==3.5.1
-numba==0.54.1
-numpy==1.20.3
+numba>=0.55.0
+numpy>=1.21.5
 outlier-utils==0.0.3
 pyarrow==6.0.1
 pandas==1.3.5

saqc/core/modules/__init__.py

@@ -17,7 +17,7 @@ from saqc.core.modules.interpolation import Interpolation
 from saqc.core.modules.outliers import Outliers
 from saqc.core.modules.pattern import Pattern
 from saqc.core.modules.resampling import Resampling
-from saqc.core.modules.residues import Residues
+from saqc.core.modules.residuals import Residuals
 from saqc.core.modules.rolling import Rolling
 from saqc.core.modules.scores import Scores
 from saqc.core.modules.tools import Tools
@@ -38,7 +38,7 @@ class FunctionsMixin(
     Outliers,
     Pattern,
     Resampling,
-    Residues,
+    Residuals,
     Rolling,
     Scores,
     Tools,

saqc/core/modules/breaks.py

@@ -9,14 +9,18 @@ from __future__ import annotations
 
 from saqc.constants import BAD, FILTER_ALL
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Breaks:
+    @doc(saqc.funcs.breaks.flagMissing.__doc__)
     def flagMissing(
         self, field: str, flag: float = BAD, dfilter: float = FILTER_ALL, **kwargs
     ) -> saqc.SaQC:
         return self._defer("flagMissing", locals())
 
+    @doc(saqc.funcs.breaks.flagIsolated.__doc__)
     def flagIsolated(
         self,
         field: str,
@@ -27,6 +31,7 @@ class Breaks:
     ) -> saqc.SaQC:
         return self._defer("flagIsolated", locals())
 
+    @doc(saqc.funcs.breaks.flagJumps.__doc__)
     def flagJumps(
         self,
         field: str,

saqc/core/modules/changepoints.py

@@ -14,9 +14,12 @@ from typing_extensions import Literal
 
 from saqc.constants import BAD
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class ChangePoints:
+    @doc(saqc.funcs.changepoints.flagChangePoints.__doc__)
     def flagChangePoints(
         self,
         field: str,
@@ -32,6 +35,7 @@ class ChangePoints:
     ) -> saqc.SaQC:
         return self._defer("flagChangePoints", locals())
 
+    @doc(saqc.funcs.changepoints.assignChangePointCluster.__doc__)
     def assignChangePointCluster(
         self,
         field: str,

saqc/core/modules/constants.py

@@ -9,9 +9,12 @@ from __future__ import annotations
 
 from saqc.constants import BAD
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Constants:
+    @doc(saqc.funcs.constants.flagByVariance.__doc__)
     def flagByVariance(
         self,
         field: str,
@@ -24,6 +27,7 @@ class Constants:
     ) -> saqc.SaQC:
         return self._defer("flagByVariance", locals())
 
+    @doc(saqc.funcs.constants.flagConstants.__doc__)
     def flagConstants(
         self, field: str, thresh: float, window: str, flag: float = BAD, **kwargs
     ) -> saqc.SaQC:

saqc/core/modules/curvefit.py

@@ -13,9 +13,12 @@ from typing_extensions import Literal
 
 from saqc.constants import BAD
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Curvefit:
+    @doc(saqc.funcs.curvefit.fitPolynomial.__doc__)
     def fitPolynomial(
         self,
         field: str,

saqc/core/modules/drift.py

@@ -17,9 +17,12 @@ from saqc.constants import BAD
 import saqc
 from saqc.funcs import LinkageString
 from saqc.lib.types import CurveFitter
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Drift:
+    @doc(saqc.funcs.drift.flagDriftFromNorm.__doc__)
     def flagDriftFromNorm(
         self,
         field: Sequence[str],
@@ -36,6 +39,7 @@ class Drift:
     ) -> saqc.SaQC:
         return self._defer("flagDriftFromNorm", locals())
 
+    @doc(saqc.funcs.drift.flagDriftFromReference.__doc__)
     def flagDriftFromReference(
         self,
         field: Sequence[str],
@@ -51,6 +55,7 @@ class Drift:
     ) -> saqc.SaQC:
         return self._defer("flagDriftFromReference", locals())
 
+    @doc(saqc.funcs.drift.correctDrift.__doc__)
     def correctDrift(
         self,
         field: str,
@@ -61,6 +66,7 @@ class Drift:
     ) -> saqc.SaQC:
         return self._defer("correctDrift", locals())
 
+    @doc(saqc.funcs.drift.correctRegimeAnomaly.__doc__)
     def correctRegimeAnomaly(
         self,
         field: str,
@@ -72,6 +78,7 @@ class Drift:
     ) -> saqc.SaQC:
         return self._defer("correctRegimeAnomaly", locals())
 
+    @doc(saqc.funcs.drift.correctOffset.__doc__)
     def correctOffset(
         self,
         field: str,

saqc/core/modules/flagtools.py

@@ -16,21 +16,28 @@ from typing_extensions import Literal
 
 from saqc.constants import BAD
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class FlagTools:
+    @doc(saqc.funcs.flagtools.clearFlags.__doc__)
     def clearFlags(self, field: str, **kwargs) -> saqc.SaQC:
         return self._defer("clearFlags", locals())
 
+    @doc(saqc.funcs.flagtools.forceFlags.__doc__)
     def forceFlags(self, field: str, flag: float = BAD, **kwargs) -> saqc.SaQC:
         return self._defer("forceFlags", locals())
 
+    @doc(saqc.funcs.flagtools.forceFlags.__doc__)
     def flagDummy(self, field: str, **kwargs) -> saqc.SaQC:
         return self._defer("flagDummy", locals())
 
+    @doc(saqc.funcs.flagtools.flagUnflagged.__doc__)
     def flagUnflagged(self, field: str, flag: float = BAD, **kwargs) -> saqc.SaQC:
         return self._defer("flagUnflagged", locals())
 
+    @doc(saqc.funcs.flagtools.flagManual.__doc__)
     def flagManual(
         self,
         field: str,
@@ -45,6 +52,7 @@ class FlagTools:
     ) -> saqc.SaQC:
         return self._defer("flagManual", locals())
 
+    @doc(saqc.funcs.flagtools.transferFlags.__doc__)
     def transferFlags(
         self,
         field: str | Sequence[str],

saqc/core/modules/generic.py

@@ -12,9 +12,12 @@ from typing import Sequence, Union
 import saqc
 from saqc.constants import UNFLAGGED, BAD, FILTER_ALL
 from saqc.lib.types import GenericFunction
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Generic:
+    @doc(saqc.funcs.generic.processGeneric.__doc__)
     def processGeneric(
         self,
         field: str | Sequence[str],
@@ -26,6 +29,7 @@ class Generic:
     ) -> saqc.SaQC:
         return self._defer("processGeneric", locals())
 
+    @doc(saqc.funcs.generic.flagGeneric.__doc__)
     def flagGeneric(
         self,
         field: Union[str, Sequence[str]],

saqc/core/modules/interpolation.py

@@ -15,9 +15,12 @@ import pandas as pd
 from saqc.constants import UNFLAGGED
 import saqc
 from saqc.funcs.interpolation import _SUPPORTED_METHODS
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Interpolation:
+    @doc(saqc.funcs.interpolation.interpolateByRolling.__doc__)
     def interpolateByRolling(
         self,
         field: str,
@@ -30,6 +33,7 @@ class Interpolation:
     ) -> saqc.SaQC:
         return self._defer("interpolateByRolling", locals())
 
+    @doc(saqc.funcs.interpolation.interpolateInvalid.__doc__)
     def interpolateInvalid(
         self,
         field: str,
@@ -42,6 +46,7 @@ class Interpolation:
     ) -> saqc.SaQC:
         return self._defer("interpolateInvalid", locals())
 
+    @doc(saqc.funcs.interpolation.interpolateIndex.__doc__)
     def interpolateIndex(
         self,
         field: str,

saqc/core/modules/noise.py

@@ -13,9 +13,12 @@ from typing import Callable
 
 from saqc.constants import BAD
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Noise:
+    @doc(saqc.funcs.noise.flagByStatLowPass.__doc__)
     def flagByStatLowPass(
         self,
         field: str,

saqc/core/modules/outliers.py

@@ -15,9 +15,12 @@ from typing_extensions import Literal
 
 from saqc.constants import BAD
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Outliers:
+    @doc(saqc.funcs.outliers.flagByStray.__doc__)
     def flagByStray(
         self,
         field: str,
@@ -30,6 +33,7 @@ class Outliers:
     ) -> saqc.SaQC:
         return self._defer("flagByStray", locals())
 
+    @doc(saqc.funcs.outliers.flagMVScores.__doc__)
     def flagMVScores(
         self,
         field: Sequence[str],
@@ -49,6 +53,7 @@ class Outliers:
     ) -> saqc.SaQC:
         return self._defer("flagMVScores", locals())
 
+    @doc(saqc.funcs.outliers.flagRaise.__doc__)
     def flagRaise(
         self,
         field: str,
@@ -64,6 +69,7 @@ class Outliers:
     ) -> saqc.SaQC:
         return self._defer("flagRaise", locals())
 
+    @doc(saqc.funcs.outliers.flagMAD.__doc__)
     def flagMAD(
         self,
         field: str,
@@ -74,6 +80,7 @@ class Outliers:
     ) -> saqc.SaQC:
         return self._defer("flagMAD", locals())
 
+    @doc(saqc.funcs.outliers.flagOffset.__doc__)
     def flagOffset(
         self,
         field: str,
@@ -86,6 +93,7 @@ class Outliers:
     ) -> saqc.SaQC:
         return self._defer("flagOffset", locals())
 
+    @doc(saqc.funcs.outliers.flagByGrubbs.__doc__)
     def flagByGrubbs(
         self,
         field: str,
@@ -98,6 +106,7 @@ class Outliers:
     ) -> saqc.SaQC:
         return self._defer("flagByGrubbs", locals())
 
+    @doc(saqc.funcs.outliers.flagRange.__doc__)
     def flagRange(
         self,
         field: str,
@@ -108,6 +117,7 @@ class Outliers:
     ) -> saqc.SaQC:
         return self._defer("flagRange", locals())
 
+    @doc(saqc.funcs.outliers.flagCrossStatistics.__doc__)
     def flagCrossStatistics(
         self,
         field: Sequence[str],
@@ -116,4 +126,4 @@ class Outliers:
         flag: float = BAD,
         **kwargs,
     ) -> saqc.SaQC:
-        return self._defer("flagCrossStatistic", locals())
+        return self._defer("flagCrossStatistics", locals())

saqc/core/modules/pattern.py

@@ -9,9 +9,12 @@ from __future__ import annotations
 
 from saqc.constants import BAD
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Pattern:
+    @doc(saqc.funcs.pattern.flagPatternByDTW.__doc__)
     def flagPatternByDTW(
         self,
         field,

saqc/core/modules/resampling.py

@@ -16,9 +16,12 @@ from typing_extensions import Literal
 from saqc.constants import BAD
 import saqc
 from saqc.funcs.interpolation import _SUPPORTED_METHODS
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Resampling:
+    @doc(saqc.funcs.resampling.linear.__doc__)
     def linear(
         self,
         field: str,
@@ -27,6 +30,7 @@ class Resampling:
     ) -> saqc.SaQC:
         return self._defer("linear", locals())
 
+    @doc(saqc.funcs.resampling.interpolate.__doc__)
     def interpolate(
         self,
         field: str,
@@ -37,6 +41,7 @@ class Resampling:
     ) -> saqc.SaQC:
         return self._defer("interpolate", locals())
 
+    @doc(saqc.funcs.resampling.shift.__doc__)
     def shift(
         self,
         field: str,
@@ -47,6 +52,7 @@ class Resampling:
     ) -> saqc.SaQC:
         return self._defer("shift", locals())
 
+    @doc(saqc.funcs.resampling.resample.__doc__)
     def resample(
         self,
         field: str,
@@ -63,6 +69,7 @@ class Resampling:
     ) -> saqc.SaQC:
         return self._defer("resample", locals())
 
+    @doc(saqc.funcs.resampling.concatFlags.__doc__)
     def concatFlags(
         self,
         field: str,

saqc/core/modules/residues.py → saqc/core/modules/residuals.py

@@ -15,10 +15,13 @@ from typing_extensions import Literal
 
 from saqc.constants import BAD
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
-class Residues:
-    def calculatePolynomialResidues(
+class Residuals:
+    @doc(saqc.funcs.residuals.calculatePolynomialResiduals.__doc__)
+    def calculatePolynomialResiduals(
         self,
         field: str,
         window: Union[str, int],
@@ -26,9 +29,10 @@ class Residues:
         min_periods: Optional[int] = 0,
         **kwargs,
     ) -> saqc.SaQC:
-        return self._defer("calculatePolynomialResidues", locals())
+        return self._defer("calculatePolynomialResiduals", locals())
 
-    def calculateRollingResidues(
+    @doc(saqc.funcs.residuals.calculateRollingResiduals.__doc__)
+    def calculateRollingResiduals(
         self,
         field: str,
         window: Union[str, int],
@@ -37,4 +41,4 @@ class Residues:
         center: bool = True,
         **kwargs,
     ) -> saqc.SaQC:
-        return self._defer("calculateRollingResidues", locals())
+        return self._defer("calculateRollingResiduals", locals())

saqc/core/modules/rolling.py

@@ -12,9 +12,12 @@ import numpy as np
 import pandas as pd
 
 from saqc.constants import BAD
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Rolling:
+    @doc(saqc.funcs.rolling.roll.__doc__)
     def roll(
         self,
         field: str,

saqc/core/modules/scores.py

@@ -14,9 +14,12 @@ import pandas as pd
 from typing_extensions import Literal
 
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Scores:
+    @doc(saqc.funcs.scores.assignKNNScore.__doc__)
     def assignKNNScore(
         self,
         field: Sequence[str],

saqc/core/modules/tools.py

@@ -14,20 +14,26 @@ import saqc
 import numpy as np
 
 from saqc.constants import FILTER_NONE
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Tools:
+    @doc(saqc.funcs.tools.copyField.__doc__)
     def copyField(
         self, field: str, target: str, overwrite: bool = False, **kwargs
     ) -> saqc.SaQC:
         return self._defer("copyField", locals())
 
+    @doc(saqc.funcs.tools.dropField.__doc__)
     def dropField(self, field: str, **kwargs) -> saqc.SaQC:
         return self._defer("dropField", locals())
 
+    @doc(saqc.funcs.tools.renameField.__doc__)
     def renameField(self, field: str, new_name: str, **kwargs) -> saqc.SaQC:
         return self._defer("renameField", locals())
 
+    @doc(saqc.funcs.tools.maskTime.__doc__)
     def maskTime(
         self,
         field: str,
@@ -40,6 +46,7 @@ class Tools:
     ) -> saqc.SaQC:
         return self._defer("maskTime", locals())
 
+    @doc(saqc.funcs.tools.plot.__doc__)
     def plot(
         self,
         field: str,

saqc/core/modules/transformation.py

@@ -12,9 +12,12 @@ from typing import Callable, Optional, Union
 import pandas as pd
 
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Transformation:
+    @doc(saqc.funcs.transformation.transform.__doc__)
     def transform(
         self,
         field: str,