From 4a190f44883fd5a4e7f5eadf785defa936d5f403 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 26 Aug 2021 12:56:46 +0200
Subject: [PATCH] [FIX] flagMissing needs to be masking="none" as we otherwise
likely flag the masked values
---
saqc/funcs/breaks.py | 13 +++++++++++--
saqc/funcs/flagtools.py | 1 +
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/saqc/funcs/breaks.py b/saqc/funcs/breaks.py
index ae6791afd..4d34a550a 100644
--- a/saqc/funcs/breaks.py
+++ b/saqc/funcs/breaks.py
@@ -20,16 +20,22 @@ from saqc.constants import *
from saqc.lib.tools import groupConsecutives
from saqc.lib.types import FreqString, ColumnName, IntegerWindow
from saqc.funcs.changepoints import assignChangePointCluster
-from saqc.core import flagging, Flags
+from saqc.core.flags import Flags
+from saqc.core.history import History
+from saqc.core.register import _isflagged, flagging
-@flagging(masking="field", module="breaks")
+# NOTE:
+# masking="none" as we otherwise might interprete
+# the masked values as missing data
+@flagging(masking="none", module="breaks")
def flagMissing(
data: DictOfSeries,
field: ColumnName,
flags: Flags,
nodata: float = np.nan,
flag: float = BAD,
+ to_mask: float = UNFLAGGED,
**kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
@@ -55,11 +61,14 @@ def flagMissing(
flags : saqc.Flags
The quality flags of data
"""
+
datacol = data[field]
+
if np.isnan(nodata):
mask = datacol.isna()
else:
mask = datacol == nodata
+ mask = ~_isflagged(flags[field], to_mask) & mask
flags[mask, field] = flag
return data, flags
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index c5553dde9..f75730b0c 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -82,6 +82,7 @@ def clearFlags(
forceFlags : set whole column to a flag value
flagUnflagged : set flag value at all unflagged positions
"""
+ # NOTE: do we really need this?
if "flag" in kwargs:
kwargs = {**kwargs} # copy
flag = kwargs.pop("flag")
--
GitLab