Skip to content
Snippets Groups Projects
Commit 9d1a02e0 authored by Bert Palm's avatar Bert Palm 🎇
Browse files

adjusted `to_mask` according to #GL160, simplified `unmaskData` and added more...

adjusted `to_mask` according to #GL160, simplified `unmaskData` and added more comments, because its so hard to wrap the brain around it, without get fu**..messed up
parent 55fb9af8
No related branches found
No related tags found
4 merge requests!271Static expansion of regular expressions,!260Follow-Up Translations,!237Flagger Translations,!232WIP: Fuzzy testing
...@@ -33,7 +33,7 @@ class CallCtrl: ...@@ -33,7 +33,7 @@ class CallCtrl:
kwargs: dict kwargs: dict
masking: MaskingStrT = None masking: MaskingStrT = None
to_mask: List[float] = None to_mask: float = None
mask: dios.DictOfSeries = None mask: dios.DictOfSeries = None
...@@ -157,7 +157,7 @@ def _getToMask(ctrl): ...@@ -157,7 +157,7 @@ def _getToMask(ctrl):
_warnForUnusedMasking(ctrl.masking, to_mask) _warnForUnusedMasking(ctrl.masking, to_mask)
if to_mask is None: if to_mask is None:
to_mask = [UNFLAGGED] to_mask = UNFLAGGED
return to_mask return to_mask
...@@ -165,9 +165,10 @@ def _getToMask(ctrl): ...@@ -165,9 +165,10 @@ def _getToMask(ctrl):
def _warnForUnusedMasking(masking, to_mask): def _warnForUnusedMasking(masking, to_mask):
# warn if the user explicitly pass `to_mask=..` to a function that is # warn if the user explicitly pass `to_mask=..` to a function that is
# decorated by `register(masking='none')`, by which `to_mask` is ignored # decorated by `register(masking='none')`, by which `to_mask` is ignored
if masking == 'none' and to_mask not in (None, []): # TODO: fix warning message
# todo: see following message if masking == 'none' and to_mask not in (None, np.inf):
logging.warning("`to_mask` is given, but the test ignore masking. Please refer to the documentation: TODO") logging.warning("`to_mask` is given, but the saqc-function ignore masking."
" Please refer to the documentation: TODO")
# TODO: this is heavily undertested # TODO: this is heavily undertested
...@@ -191,22 +192,12 @@ def _maskData(data, flagger, columns, to_mask) -> Tuple[dios.DictOfSeries, dios. ...@@ -191,22 +192,12 @@ def _maskData(data, flagger, columns, to_mask) -> Tuple[dios.DictOfSeries, dios.
return data, mask return data, mask
# todo: solve with outcome of #GL160 def _getMask(flags: Union[np.array, pd.Series], to_mask: float) -> Union[np.array, pd.Series]:
def _getMask(flags: Union[np.array, pd.Series], to_mask: list) -> Union[np.array, pd.Series]:
""" """
Return a mask of flags accordingly to `to_mask`. Return a mask of flags accordingly to `to_mask`.
Return type is same as flags. Return type is same as flags.
""" """
return flags > to_mask
if isinstance(flags, pd.Series):
mask = pd.Series(False, index=flags.index, dtype=bool)
else:
mask = np.zeros_like(flags, dtype=bool)
for f in to_mask:
mask |= flags == f
return ~mask
def _prepareFlags(flagger: Flagger, ctrl: CallCtrl) -> Flagger: def _prepareFlags(flagger: Flagger, ctrl: CallCtrl) -> Flagger:
...@@ -274,15 +265,23 @@ def _unmaskData(data: dios.DictOfSeries, ctrl: CallCtrl) -> dios.DictOfSeries: ...@@ -274,15 +265,23 @@ def _unmaskData(data: dios.DictOfSeries, ctrl: CallCtrl) -> dios.DictOfSeries:
for c in columns: for c in columns:
# ignore
if old.data[c].empty or data[c].empty or old.mask[c].empty: if old.data[c].empty or data[c].empty or old.mask[c].empty:
continue continue
if old.data[c].index.equals(data[c].index): # on index changed, we simply ignore the old data
restore_old_val = old.mask[c].to_numpy() & data[c].isna().to_numpy() if not old.data[c].index.equals(data[c].index):
continue
restore_old_val = old.mask[c].to_numpy() & data[c].isna().to_numpy()
# we have nothing to restore
if not any(restore_old_val):
continue
if any(restore_old_val): # restore old values if no new are present
ol, nw = old.data[c].to_numpy(), data[c].to_numpy() ol, nw = old.data[c].to_numpy(), data[c].to_numpy()
data.loc[:, c] = np.where(restore_old_val, ol, nw) data.loc[:, c] = np.where(restore_old_val, ol, nw)
return data return data
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment