From 9d1a02e0a965ab9dcd55fa5f073b6a21cb9fbfca Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Mon, 1 Mar 2021 23:49:11 +0100
Subject: [PATCH] adjusted `to_mask` according to #GL160, simplified
`unmaskData` and added more comments, because its so hard to wrap the brain
around it, without get fu**..messed up
---
saqc/core/register.py | 43 +++++++++++++++++++++----------------------
1 file changed, 21 insertions(+), 22 deletions(-)
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 5493deba2..fcdcf5049 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -33,7 +33,7 @@ class CallCtrl:
kwargs: dict
masking: MaskingStrT = None
- to_mask: List[float] = None
+ to_mask: float = None
mask: dios.DictOfSeries = None
@@ -157,7 +157,7 @@ def _getToMask(ctrl):
_warnForUnusedMasking(ctrl.masking, to_mask)
if to_mask is None:
- to_mask = [UNFLAGGED]
+ to_mask = UNFLAGGED
return to_mask
@@ -165,9 +165,10 @@ def _getToMask(ctrl):
def _warnForUnusedMasking(masking, to_mask):
# warn if the user explicitly pass `to_mask=..` to a function that is
# decorated by `register(masking='none')`, by which `to_mask` is ignored
- if masking == 'none' and to_mask not in (None, []):
- # todo: see following message
- logging.warning("`to_mask` is given, but the test ignore masking. Please refer to the documentation: TODO")
+ # TODO: fix warning message
+ if masking == 'none' and to_mask not in (None, np.inf):
+ logging.warning("`to_mask` is given, but the saqc-function ignore masking."
+ " Please refer to the documentation: TODO")
# TODO: this is heavily undertested
@@ -191,22 +192,12 @@ def _maskData(data, flagger, columns, to_mask) -> Tuple[dios.DictOfSeries, dios.
return data, mask
-# todo: solve with outcome of #GL160
-def _getMask(flags: Union[np.array, pd.Series], to_mask: list) -> Union[np.array, pd.Series]:
+def _getMask(flags: Union[np.array, pd.Series], to_mask: float) -> Union[np.array, pd.Series]:
"""
Return a mask of flags accordingly to `to_mask`.
Return type is same as flags.
"""
-
- if isinstance(flags, pd.Series):
- mask = pd.Series(False, index=flags.index, dtype=bool)
- else:
- mask = np.zeros_like(flags, dtype=bool)
-
- for f in to_mask:
- mask |= flags == f
-
- return ~mask
+ return flags > to_mask
def _prepareFlags(flagger: Flagger, ctrl: CallCtrl) -> Flagger:
@@ -274,15 +265,23 @@ def _unmaskData(data: dios.DictOfSeries, ctrl: CallCtrl) -> dios.DictOfSeries:
for c in columns:
+ # ignore
if old.data[c].empty or data[c].empty or old.mask[c].empty:
continue
- if old.data[c].index.equals(data[c].index):
- restore_old_val = old.mask[c].to_numpy() & data[c].isna().to_numpy()
+ # on index changed, we simply ignore the old data
+ if not old.data[c].index.equals(data[c].index):
+ continue
+
+ restore_old_val = old.mask[c].to_numpy() & data[c].isna().to_numpy()
+
+ # we have nothing to restore
+ if not any(restore_old_val):
+ continue
- if any(restore_old_val):
- ol, nw = old.data[c].to_numpy(), data[c].to_numpy()
- data.loc[:, c] = np.where(restore_old_val, ol, nw)
+ # restore old values if no new are present
+ ol, nw = old.data[c].to_numpy(), data[c].to_numpy()
+ data.loc[:, c] = np.where(restore_old_val, ol, nw)
return data
--
GitLab