Merge branch 'propagateFlags-policy' into 'develop'

make propagateFlags policiy compliant

See merge request !695

CHANGELOG.md

@@ -19,6 +19,7 @@ SPDX-License-Identifier: GPL-3.0-or-later
 ### Fixed
 - Bug in `SaQC.assignChangePointCluster` and `SaQC.flagChangePoints`: A tuple passed 
   to `min_period` was only recognised if also `window` was a tuple.
+- `SaQC.propagateFlags` was overwriting existing flags
 ### Deprecated
 
 ## [2.4.1](https://git.ufz.de/rdm-software/saqc/-/tags/v2.4.1) - 2023-06-22

saqc/funcs/flagtools.py

@@ -18,7 +18,7 @@ from typing_extensions import Literal
 from saqc import BAD, FILTER_ALL, UNFLAGGED
 from saqc.core import DictOfSeries, flagging, register
 from saqc.lib.checking import validateChoice, validateWindow
-from saqc.lib.tools import isflagged, toSequence
+from saqc.lib.tools import isflagged, isunflagged, toSequence
 
 if TYPE_CHECKING:
     from saqc import SaQC
@@ -458,7 +458,7 @@ class FlagtoolsMixin:
         # consider everything != np.nan as flag
         flagged = isflagged(hc, dfilter)
 
-        repeated = (
+        mask = (
             flagged.rolling(window, min_periods=1, closed="left")
             .max()
             .fillna(0)
@@ -466,9 +466,10 @@ class FlagtoolsMixin:
         )
 
         if method == "bfill":
-            repeated = repeated[::-1]
+            mask = mask[::-1]
+        mask = isunflagged(self._flags[field], thresh=dfilter) & mask
 
-        self._flags[repeated, field] = flag
+        self._flags[mask, field] = flag
 
         return self
 

tests/funcs/test_flagtools.py

@@ -44,23 +44,23 @@ N = np.nan
         ([B, U, U, U, U], [N, N, N, N, N], {"window": "10D", "method": "bfill"}),
         # playing with dfilter
         (
-            [1, 1, B, 1, 1],
-            [N, B, B, B, B],
+            [1, B, -1, -1, -1],
+            [N, N, B, B, N],
             {"window": 2, "method": "ffill", "dfilter": 0},
         ),
         (
-            [1, 1, B, 1, 1],
-            [B, B, B, B, N],
+            [-1, -1, -1, B, 1],
+            [N, B, B, N, N],
             {"window": 2, "method": "bfill", "dfilter": 0},
         ),
         (
-            [B, 1, 1, 1, 1],
-            [N, B, B, B, B],
+            [B, 1, -1, 1, 1],
+            [N, N, B, N, N],
             {"window": "2D", "method": "ffill", "dfilter": 0},
         ),
         (
-            [B, 1, 1, 1, 1],
-            [B, B, B, B, N],
+            [B, 1, 1, -1, 1],
+            [N, N, N, B, N],
             {"window": "2D", "method": "bfill", "dfilter": 0},
         ),
     ],
@@ -71,7 +71,7 @@ def test_propagateFlagsRegularIndex(got, expected, kwargs):
     expected = pd.Series(expected, index=index)
     data = pd.DataFrame({"x": np.nan}, index=index)
     saqc = SaQC(data=data, flags=flags).propagateFlags(field="x", **kwargs)
-    result = saqc._flags.history["x"].hist[1].astype(float)
+    result = saqc._history["x"].hist[1].astype(float)
     assert result.equals(expected)