.gitlab-ci.yml

@@ -80,7 +80,7 @@ coverage:
     - export DISPLAY=:99
     - Xvfb :99 &
     - pip install pytest-cov coverage
-    - pytest --cov=saqc tests --ignore=tests/fuzzy -Werror
+    - pytest --cov=saqc tests --ignore=tests/fuzzy tests/extras -Werror
   after_script:
     - coverage xml
   # regex to find the coverage percentage in the job output
@@ -99,7 +99,7 @@ python39:
   script:
     - export DISPLAY=:99
     - Xvfb :99 &
-    - pytest tests -Werror --junitxml=report.xml
+    - pytest tests -Werror --junitxml=report.xml --ignore=tests/extras
     - python -m saqc --config docs/resources/data/config.csv --data docs/resources/data/data.csv --outfile /tmp/test.csv
   artifacts:
     when: always
@@ -113,7 +113,7 @@ python310:
   script:
     - export DISPLAY=:99
     - Xvfb :99 &
-    - pytest tests -Werror --junitxml=report.xml
+    - pytest tests -Werror --junitxml=report.xml --ignore=tests/extras
     - python -m saqc --config docs/resources/data/config.csv --data docs/resources/data/data.csv --outfile /tmp/test.csv
   artifacts:
     when: always
@@ -126,7 +126,7 @@ python311:
   script:
     - export DISPLAY=:99
     - Xvfb :99 &
-    - pytest tests -Werror --junitxml=report.xml
+    - pytest tests -Werror --junitxml=report.xml --ignore=tests/extras
     - python -m saqc --config docs/resources/data/config.csv --data docs/resources/data/data.csv --outfile /tmp/test.csv
   artifacts:
     when: always
@@ -139,7 +139,7 @@ python312:
   script:
     - export DISPLAY=:99
     - Xvfb :99 &
-    - pytest tests -Werror --junitxml=report.xml
+    - pytest tests -Werror --junitxml=report.xml --ignore=tests/extras
     - python -m saqc --config docs/resources/data/config.csv --data docs/resources/data/data.csv --outfile /tmp/test.csv
   artifacts:
     when: always

CHANGELOG.md

@@ -8,9 +8,14 @@ SPDX-License-Identifier: GPL-3.0-or-later
 ## Unreleased
 [List of commits](https://git.ufz.de/rdm-software/saqc/-/compare/v2.6.0...develop)
 ### Added
+- `flagPlateaus`: added function to search and flag outlierish value plateaus of certain temporal extension
+- `flagUniLOF`: added dispatch to Local Outlier Probability (*LoOP*) variant
+- `flaguniLOF`: made `thresh` Optional
+- `flagPlateaus`: added function to search and flag anomalous value plateaus of certain temporal extension
 ### Changed
 ### Removed
 ### Fixed
+- `flagConstants`: fixed bug where last `min_periods` will never get flagged
 ### Deprecated
 
 ## [2.6.0](https://git.ufz.de/rdm-software/saqc/-/tags/v2.6.0) - 2024-04-15
@@ -27,6 +32,7 @@ SPDX-License-Identifier: GPL-3.0-or-later
 - `setFlags`: function to replace `flagManual`
 - `flagUniLOF`: added parameter `slope_correct` to correct for overflagging at relatively steep data value slopes
 - `History`: added option to change aggregation behavior
+- "horizontal" axis / multivariate mode for `rolling`
 - Translation scheme `AnnotatedFloatScheme`
 ### Changed
 - `SaQC.flags` always returns a `DictOfSeries`

docs/funcs/basicAnomalies.rst

@@ -16,5 +16,6 @@ Basic Anomalies
    ~SaQC.flagRaise
    ~SaQC.flagConstants
    ~SaQC.flagByVariance
+   ~SaQC.flagPlateau
 
 

docs/funcs/flagTools.rst

@@ -15,3 +15,5 @@ Flagtools
    ~SaQC.flagManual
    ~SaQC.flagDummy
    ~SaQC.transferFlags
+   ~SaQC.andGroup
+   ~SaQC.orGroup

docs/funcs/genericWrapper.rst

@@ -13,6 +13,6 @@ Generic Functions
 
    ~SaQC.processGeneric
    ~SaQC.flagGeneric
-   ~SaQC.rolling
-   ~SaQC.transform
-   ~SaQC.resample
+   ~SaQC.andGroup
+   ~SaQC.orGroup
+

docs/funcs/outlierDetection.rst

@@ -19,4 +19,5 @@ Univariate Outlier Detection
    ~SaQC.flagRange
    ~SaQC.flagLOF
    ~SaQC.flagZScore
+   ~SaQC.flagPlateau
 

docs/funcs/tools.rst

@@ -15,3 +15,4 @@ Tools
    ~SaQC.renameField
    ~SaQC.selectTime
    ~SaQC.plot
+

docs/resources/data/turbidity_plateaus.csv.license

+SPDX-FileCopyrightText: 2021 Helmholtz-Zentrum für Umweltforschung GmbH - UFZ
+
+SPDX-License-Identifier: GPL-3.0-or-later
\ No newline at end of file

docs/resources/images/fitFMpic.png.license

+SPDX-FileCopyrightText: 2021 Helmholtz-Zentrum für Umweltforschung GmbH - UFZ
+
+SPDX-License-Identifier: GPL-3.0-or-later
\ No newline at end of file

docs/resources/images/horizontalAxisRollingExample.png.license

+SPDX-FileCopyrightText: 2021 Helmholtz-Zentrum für Umweltforschung GmbH - UFZ
+
+SPDX-License-Identifier: GPL-3.0-or-later
\ No newline at end of file

saqc/__init__.py

@@ -7,13 +7,34 @@
 
 """The System for automated Quality Control package."""
 
+__all__ = [
+    "BAD",
+    "DOUBTFUL",
+    "GOOD",
+    "UNFLAGGED",
+    "FILTER_ALL",
+    "FILTER_NONE",
+    "Flags",
+    "DictOfSeries",
+    "SaQC",
+    "DmpScheme",
+    "FloatScheme",
+    "PositionalScheme",
+    "SimpleScheme",
+    "AnnotatedFloatScheme",
+    "fromConfig",
+]
+
 from saqc.constants import BAD, DOUBTFUL, FILTER_ALL, FILTER_NONE, GOOD, UNFLAGGED
-from saqc.exceptions import ParsingError
 from saqc.core import Flags, DictOfSeries, SaQC
-from saqc.core.translation import DmpScheme, FloatScheme, PositionalScheme, SimpleScheme
+from saqc.core.translation import (
+    DmpScheme,
+    FloatScheme,
+    PositionalScheme,
+    SimpleScheme,
+    AnnotatedFloatScheme,
+)
 from saqc.parsing.reader import fromConfig
-from saqc.version import __version__
-
 from . import _version
 
 __version__ = _version.get_versions()["version"]

saqc/__main__.py

@@ -136,10 +136,10 @@ def main(
     config = str(config)
     cr = _ConfigReader(data=data, scheme=scheme)
     if config.endswith("json"):
-        f = None
-        if json_field is not None:
-            f = lambda j: j[str(json_field)]
-        cr = cr.readJson(config, unpack=f)
+        cr = cr.readJson(
+            config,
+            unpack=lambda j: j[str(json_field)] if json_field is not None else None,
+        )
     else:
         cr = cr.readCsv(config)
 

saqc/core/__init__.py

@@ -4,8 +4,19 @@
 # -*- coding: utf-8 -*-
 
 # isort: skip_file
+
+__all__ = [
+    "DictOfSeries",
+    "History",
+    "Flags",
+    "SaQC",
+    "flagging",
+    "processing",
+    "register",
+]
+
 from saqc.core.frame import DictOfSeries
 from saqc.core.history import History
-from saqc.core.flags import Flags, initFlagsLike
+from saqc.core.flags import Flags
 from saqc.core.register import flagging, processing, register
 from saqc.core.core import SaQC

saqc/core/core.py

@@ -12,7 +12,7 @@ import warnings
 from copy import copy as shallowcopy
 from copy import deepcopy
 from functools import partial
-from typing import Any, Hashable, Iterable, MutableMapping
+from typing import Any, Hashable, Iterable
 
 import numpy as np
 import pandas as pd
@@ -22,6 +22,7 @@ from saqc.core.frame import DictOfSeries
 from saqc.core.history import History
 from saqc.core.register import FUNC_MAP
 from saqc.core.translation import (
+    AnnotatedFloatScheme,
     DmpScheme,
     FloatScheme,
     PositionalScheme,
@@ -41,6 +42,7 @@ TRANSLATION_SCHEMES = {
     "float": FloatScheme,
     "dmp": DmpScheme,
     "positional": PositionalScheme,
+    "annotated-float": AnnotatedFloatScheme,
 }
 
 
@@ -330,7 +332,7 @@ class SaQC(FunctionsMixin):
     def _castData(self, data) -> DictOfSeries:
         if isinstance(data, pd.Series):
             if not isinstance(data.name, str):
-                raise ValueError(f"Cannot init from unnamed pd.Series")
+                raise ValueError("Cannot init from unnamed pd.Series")
             data = data.to_frame()
         if isinstance(data, pd.DataFrame):
             for idx in [data.index, data.columns]:

saqc/core/flags.py

@@ -7,7 +7,6 @@
 from __future__ import annotations
 
 import typing
-import warnings
 from typing import DefaultDict, Dict, Iterable, Mapping, Tuple, Type, Union, overload
 
 import numpy as np
@@ -243,7 +242,7 @@ class Flags:
         if history.empty:
             return history
 
-        errm = f"History "
+        errm = "History "
         if colname:
             errm += f"of column {colname} "
 

saqc/core/translation/__init__.py

@@ -5,6 +5,16 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 # -*- coding: utf-8 -*-
+__all__ = [
+    "MappingScheme",
+    "TranslationScheme",
+    "DmpScheme",
+    "AnnotatedFloatScheme",
+    "FloatScheme",
+    "PositionalScheme",
+    "SimpleScheme",
+]
+
 from saqc.core.translation.basescheme import MappingScheme, TranslationScheme
 from saqc.core.translation.dmpscheme import DmpScheme
 from saqc.core.translation.floatscheme import AnnotatedFloatScheme, FloatScheme

saqc/core/translation/dmpscheme.py

@@ -9,7 +9,6 @@
 from __future__ import annotations
 
 import json
-from functools import reduce
 
 import numpy as np
 import pandas as pd
@@ -18,7 +17,6 @@ from saqc import BAD, DOUBTFUL, GOOD, UNFLAGGED
 from saqc.core import Flags, History
 from saqc.core.frame import DictOfSeries
 from saqc.core.translation.basescheme import BackwardMap, ForwardMap, MappingScheme
-from saqc.lib.tools import getUnionIndex
 
 _QUALITY_CAUSES = [
     "",