From d642ffddb8de5812b9995ad3b2ff17dc6a23efd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?David=20Sch=C3=A4fer?= <david.schaefer@ufz.de>
Date: Tue, 11 Jul 2023 10:34:29 +0200
Subject: [PATCH] make propagateFlags policiy compliant
---
CHANGELOG.md | 1 +
saqc/funcs/flagtools.py | 9 +++++----
tests/funcs/test_flagtools.py | 18 +++++++++---------
3 files changed, 15 insertions(+), 13 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4bb4402a4..742657d3c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,7 @@ SPDX-License-Identifier: GPL-3.0-or-later
### Fixed
- Bug in `SaQC.assignChangePointCluster` and `SaQC.flagChangePoints`: A tuple passed
to `min_period` was only recognised if also `window` was a tuple.
+- `SaQC.propagateFlags` was overwriting existing flags
### Deprecated
## [2.4.1](https://git.ufz.de/rdm-software/saqc/-/tags/v2.4.1) - 2023-06-22
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index 99dbd59b1..049363315 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -18,7 +18,7 @@ from typing_extensions import Literal
from saqc import BAD, FILTER_ALL, UNFLAGGED
from saqc.core import DictOfSeries, flagging, register
from saqc.lib.checking import validateChoice, validateWindow
-from saqc.lib.tools import isflagged, toSequence
+from saqc.lib.tools import isflagged, isunflagged, toSequence
if TYPE_CHECKING:
from saqc import SaQC
@@ -458,7 +458,7 @@ class FlagtoolsMixin:
# consider everything != np.nan as flag
flagged = isflagged(hc, dfilter)
- repeated = (
+ mask = (
flagged.rolling(window, min_periods=1, closed="left")
.max()
.fillna(0)
@@ -466,9 +466,10 @@ class FlagtoolsMixin:
)
if method == "bfill":
- repeated = repeated[::-1]
+ mask = mask[::-1]
+ mask = isunflagged(self._flags[field], thresh=dfilter) & mask
- self._flags[repeated, field] = flag
+ self._flags[mask, field] = flag
return self
diff --git a/tests/funcs/test_flagtools.py b/tests/funcs/test_flagtools.py
index a27f3e18b..e1c4a65dd 100644
--- a/tests/funcs/test_flagtools.py
+++ b/tests/funcs/test_flagtools.py
@@ -44,23 +44,23 @@ N = np.nan
([B, U, U, U, U], [N, N, N, N, N], {"window": "10D", "method": "bfill"}),
# playing with dfilter
(
- [1, 1, B, 1, 1],
- [N, B, B, B, B],
+ [1, B, -1, -1, -1],
+ [N, N, B, B, N],
{"window": 2, "method": "ffill", "dfilter": 0},
),
(
- [1, 1, B, 1, 1],
- [B, B, B, B, N],
+ [-1, -1, -1, B, 1],
+ [N, B, B, N, N],
{"window": 2, "method": "bfill", "dfilter": 0},
),
(
- [B, 1, 1, 1, 1],
- [N, B, B, B, B],
+ [B, 1, -1, 1, 1],
+ [N, N, B, N, N],
{"window": "2D", "method": "ffill", "dfilter": 0},
),
(
- [B, 1, 1, 1, 1],
- [B, B, B, B, N],
+ [B, 1, 1, -1, 1],
+ [N, N, N, B, N],
{"window": "2D", "method": "bfill", "dfilter": 0},
),
],
@@ -71,7 +71,7 @@ def test_propagateFlagsRegularIndex(got, expected, kwargs):
expected = pd.Series(expected, index=index)
data = pd.DataFrame({"x": np.nan}, index=index)
saqc = SaQC(data=data, flags=flags).propagateFlags(field="x", **kwargs)
- result = saqc._flags.history["x"].hist[1].astype(float)
+ result = saqc._history["x"].hist[1].astype(float)
assert result.equals(expected)
--
GitLab