diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bb4402a496a47a8cb34fb17424e506884f3342c..742657d3ce5ab010ac3a7638abdb1c1ef1132f83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ SPDX-License-Identifier: GPL-3.0-or-later ### Fixed - Bug in `SaQC.assignChangePointCluster` and `SaQC.flagChangePoints`: A tuple passed to `min_period` was only recognised if also `window` was a tuple. +- `SaQC.propagateFlags` was overwriting existing flags ### Deprecated ## [2.4.1](https://git.ufz.de/rdm-software/saqc/-/tags/v2.4.1) - 2023-06-22 diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py index 99dbd59b13c37194bd2df5abb634355ee7325965..049363315571be0a9e3d636e242423f3e9656463 100644 --- a/saqc/funcs/flagtools.py +++ b/saqc/funcs/flagtools.py @@ -18,7 +18,7 @@ from typing_extensions import Literal from saqc import BAD, FILTER_ALL, UNFLAGGED from saqc.core import DictOfSeries, flagging, register from saqc.lib.checking import validateChoice, validateWindow -from saqc.lib.tools import isflagged, toSequence +from saqc.lib.tools import isflagged, isunflagged, toSequence if TYPE_CHECKING: from saqc import SaQC @@ -458,7 +458,7 @@ class FlagtoolsMixin: # consider everything != np.nan as flag flagged = isflagged(hc, dfilter) - repeated = ( + mask = ( flagged.rolling(window, min_periods=1, closed="left") .max() .fillna(0) @@ -466,9 +466,10 @@ class FlagtoolsMixin: ) if method == "bfill": - repeated = repeated[::-1] + mask = mask[::-1] + mask = isunflagged(self._flags[field], thresh=dfilter) & mask - self._flags[repeated, field] = flag + self._flags[mask, field] = flag return self diff --git a/tests/funcs/test_flagtools.py b/tests/funcs/test_flagtools.py index a27f3e18b4bbe402d88a1e3c71faf4921afd9791..e1c4a65ddaa42ea2053a468971b06aff92522003 100644 --- a/tests/funcs/test_flagtools.py +++ b/tests/funcs/test_flagtools.py @@ -44,23 +44,23 @@ N = np.nan ([B, U, U, U, U], [N, N, N, N, N], {"window": "10D", "method": "bfill"}), # playing with dfilter ( - [1, 1, B, 1, 1], - [N, B, B, B, B], + [1, B, -1, -1, -1], + [N, N, B, B, N], {"window": 2, "method": "ffill", "dfilter": 0}, ), ( - [1, 1, B, 1, 1], - [B, B, B, B, N], + [-1, -1, -1, B, 1], + [N, B, B, N, N], {"window": 2, "method": "bfill", "dfilter": 0}, ), ( - [B, 1, 1, 1, 1], - [N, B, B, B, B], + [B, 1, -1, 1, 1], + [N, N, B, N, N], {"window": "2D", "method": "ffill", "dfilter": 0}, ), ( - [B, 1, 1, 1, 1], - [B, B, B, B, N], + [B, 1, 1, -1, 1], + [N, N, N, B, N], {"window": "2D", "method": "bfill", "dfilter": 0}, ), ], @@ -71,7 +71,7 @@ def test_propagateFlagsRegularIndex(got, expected, kwargs): expected = pd.Series(expected, index=index) data = pd.DataFrame({"x": np.nan}, index=index) saqc = SaQC(data=data, flags=flags).propagateFlags(field="x", **kwargs) - result = saqc._flags.history["x"].hist[1].astype(float) + result = saqc._history["x"].hist[1].astype(float) assert result.equals(expected)