added annotated float scheme to show off the History possibilities

CHANGELOG.md

@@ -20,6 +20,7 @@ SPDX-License-Identifier: GPL-3.0-or-later
 - `setFlags`: function to replace `flagManual`
 - `flagUniLOF`: added defaultly applied correction to mitigate phenomenon of overflagging at relatively steep data value slopes. (parameter `slope_correct`).
 - `History`: added option to change aggregation behavior
+- Translation scheme `FloatScheme`
 ### Changed
 ### Removed
 ### Fixed
@@ -31,6 +32,7 @@ SPDX-License-Identifier: GPL-3.0-or-later
 - `flagManual` in favor of `setFlags`
 - `flagRaise` with delegation to better replacements `flagZScore`, `flagUniLOF`, `flagJumps` or `flagOffset`
 - `flagByGrubbs` with delegation to better replacements `flagZScore`, `flagUniLOF`s
+
 ## [2.5.0](https://git.ufz.de/rdm-software/saqc/-/tags/v2.4.1) - 2023-06-22
 [List of commits](https://git.ufz.de/rdm-software/saqc/-/compare/v2.4.1...v2.5.0)
 ### Added

saqc/core/translation/__init__.py

@@ -6,10 +6,10 @@
 
 # -*- coding: utf-8 -*-
 from saqc.core.translation.basescheme import (
-    FloatScheme,
     MappingScheme,
     TranslationScheme,
 )
 from saqc.core.translation.dmpscheme import DmpScheme
 from saqc.core.translation.positionalscheme import PositionalScheme
 from saqc.core.translation.simplescheme import SimpleScheme
+from saqc.core.translation.floatscheme import FloatScheme, AnnotatedFloatScheme

saqc/core/translation/basescheme.py

@@ -215,31 +215,3 @@ class MappingScheme(TranslationScheme):
         out = self._translate(flags, self._backward)
         out.attrs = attrs or {}
         return out
-
-
-class FloatScheme(TranslationScheme):
-    """
-    Acts as the default Translator, provides a changeable subset of the
-    internal float flags
-    """
-
-    DFILTER_DEFAULT: float = FILTER_ALL
-
-    def __call__(self, flag: float | int) -> float:
-        try:
-            return float(flag)
-        except (TypeError, ValueError, OverflowError):
-            raise ValueError(f"invalid flag, expected a numerical value, got: {flag}")
-
-    def toInternal(self, flags: pd.DataFrame | DictOfSeries) -> Flags:
-        try:
-            return Flags(flags.astype(float))
-        except (TypeError, ValueError, OverflowError):
-            raise ValueError(
-                f"invalid flag(s), expected a collection of numerical values, got: {flags}"
-            )
-
-    def toExternal(self, flags: Flags, attrs: dict | None = None) -> DictOfSeries:
-        out = DictOfSeries(flags)
-        out.attrs = attrs or {}
-        return out

saqc/core/translation/floatscheme.py

+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from matplotlib._api import kwarg_error
+import numpy as np
+import pandas as pd
+
+from saqc.constants import UNFLAGGED
+from saqc.core.flags import Flags
+from saqc.core.frame import DictOfSeries
+from saqc.core.history import History
+from saqc.core.translation.basescheme import MappingScheme
+from saqc.lib.tools import getUnionIndex
+
+
+class FloatScheme(MappingScheme):
+    """
+    Acts as the default Translator, provides a changeable subset of the
+    internal float flags
+    """
+
+    _MAP = {
+        np.nan: np.nan,
+        -np.inf: -np.inf,
+        **{k: k for k in np.arange(0, 256, dtype=float)},
+    }
+
+    def __init__(self):
+        super().__init__(forward=self._MAP, backward=self._MAP)
+
+
+class AnnotatedFloatScheme(FloatScheme):
+    def toExternal(self, flags: Flags, attrs: dict | None = None) -> DictOfSeries:
+
+        tflags = super().toExternal(flags, attrs=attrs)
+
+        out = DictOfSeries()
+        for field in tflags.columns:
+            df = pd.DataFrame(
+                {
+                    "flag": tflags[field],
+                    "func": "",
+                    "parameters": "",
+                }
+            )
+
+            history = flags.history[field]
+
+            for col in history.columns:
+                valid = (history.hist[col] != UNFLAGGED) & history.hist[col].notna()
+                meta = history.meta[col]
+                df.loc[valid, "func"] = meta["func"]
+                df.loc[valid, "parameters"] = str(meta["kwargs"])
+                out[field] = df
+
+        return out
+
+    def toInternal(self, flags: DictOfSeries) -> Flags:
+        data = {}
+        for key, frame in flags.items():
+            history = History(index=frame.index)
+            for (flag, func, kwargs), values in frame.groupby(["flag", "func", "parameters"]):
+                column = pd.Series(np.nan, index=frame.index)
+                column.loc[values.index] = self(flag)
+                history.append(column, meta={"func": func, "kwargs": kwargs})
+            data[key] = history
+        return Flags(data)

tests/core/test_translator.py

@@ -16,6 +16,7 @@ import pytest
 from saqc import BAD, DOUBTFUL, FILTER_NONE, UNFLAGGED, SaQC
 from saqc.core import Flags
 from saqc.core.translation import DmpScheme, MappingScheme, PositionalScheme
+from saqc.core.translation.floatscheme import AnnotatedFloatScheme
 from tests.common import initData
 
 
@@ -277,3 +278,25 @@ def test_positionalMulitcallsPreserveState():
         expected = tflags1[k].str.slice(start=1) * 2
         got = tflags2[k].str.slice(start=1)
         assert expected.equals(got)
+
+
+def test_annotatedFloatScheme():
+    data = initData(1)
+    col = data.columns[0]
+
+    scheme = AnnotatedFloatScheme()
+    saqc = SaQC(data=data, scheme=scheme)
+    saqc = saqc.setFlags(col, data=data[col].index[::4], flag=DOUBTFUL).flagRange(
+        col, min=3, max=10, flag=BAD
+    )
+    flags = saqc.flags
+
+    qfunc = pd.DataFrame({k: v["func"] for k, v in flags.items()})
+
+    assert flags[col]["flag"].isin({DOUBTFUL, BAD, UNFLAGGED}).all(axis=None)
+    assert flags[col]["func"].isin({"", "setFlags", "flagRange"}).all(axis=None)
+
+    round_trip = scheme.toExternal(scheme.toInternal(flags))
+    assert tuple(round_trip.keys()) == tuple(flags.keys())
+    for key in flags.keys():
+        assert round_trip[key].equals(flags[key])

tests/funcs/test_flagtools.py

@@ -48,24 +48,24 @@ N = np.nan
         ([B, U, U, U, U], [N, N, N, N, N], {"window": "10D", "method": "bfill"}),
         # playing with dfilter
         (
-            [1, B, -1, -1, -1],
+            [2, B, 0, 0, 0],
             [N, N, B, B, N],
-            {"window": 2, "method": "ffill", "dfilter": 0},
+            {"window": 2, "method": "ffill", "dfilter": 1},
         ),
         (
-            [-1, -1, -1, B, 1],
+            [0, 0, 0, B, 2],
             [N, B, B, N, N],
-            {"window": 2, "method": "bfill", "dfilter": 0},
+            {"window": 2, "method": "bfill", "dfilter": 1},
         ),
         (
-            [B, 1, -1, 1, 1],
+            [B, 2, 0, 2, 2],
             [N, N, B, N, N],
-            {"window": "2D", "method": "ffill", "dfilter": 0},
+            {"window": "2D", "method": "ffill", "dfilter": 1},
         ),
         (
-            [B, 1, 1, -1, 1],
+            [B, 2, 2, 0, 2],
             [N, N, N, B, N],
-            {"window": "2D", "method": "bfill", "dfilter": 0},
+            {"window": "2D", "method": "bfill", "dfilter": 1},
         ),
     ],
 )