diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py index 3b59e65adda67f946790ee04cabd427c45432e77..d2bd79defccb45d645afc10aa2bec279987a764d 100644 --- a/saqc/flagger/flags.py +++ b/saqc/flagger/flags.py @@ -356,18 +356,33 @@ def applyFunctionOnHistory( Parameters ---------- - flags : - column : - hist_func : - hist_kws : - mask_func : - mask_kws : - last_column : - func_handle_df : + flags : Flags + Flags object holding the History in question + column : str + name of the column holding the history in question + hist_func : callable + function to apply on `History.hist` (flags) + hist_kws : dict + hist-function keywords dict + mask_func : callable + function to apply on `History.mask` (force mask) + mask_kws : dict + mask-function keywords dict + last_column : pd.Series or None, default None + The last column to apply. If None, no extra column is appended. + func_handle_df : bool + If `True`, the whole History{.hist, .mask} are passed to the given functions, thus the + function must handle `pd.Dataframes` as first input. If `False`, each column is passed + separately, thus the functions must handle those. + + Notes + ----- + After the functions are called, all `NaN`'s in `History.mask` are replaced with `False`, + and the `.mask` is casted to bool, to ensure a consistent History. Returns ------- - + Copy of Flags with altered History (in column) """ flags = flags.copy() history = flags.history[column] @@ -398,32 +413,5 @@ def applyFunctionOnHistory( return flags -def appendHistory(flags: Flags, column, append_hist): - """ - Function, specialized for used in deharm context. - - - Parameters - ---------- - flags - field - source - merge_func - merge_func_kws - last_column - - Returns - ------- - - """ - flags = flags.copy() - new_history = flags.history[column] - for app_k in [k for k in append_hist.columns if k not in new_history.columns]: - new_history.hist[app_k] = append_hist.hist[app_k] - new_history.mask[app_k] = append_hist.mask[app_k] - flags.history[column] = new_history - return flags - - # for now we keep this name Flagger = Flags diff --git a/saqc/flagger/history.py b/saqc/flagger/history.py index 2acc8f22e7785ad0fc9d00a9b088cfecad9cae51..011a2dd41bfb67bd71292e4d9bb7d58a8b6b9dce 100644 --- a/saqc/flagger/history.py +++ b/saqc/flagger/history.py @@ -393,3 +393,45 @@ class History: raise ValueError('dtype must be float') return obj + + +def appendNewerHistory(original: History, newer: History) -> History: + """ + Append all newer columns of a history to an other History. + + The first N columns in the newer History are discarded, where N is the + number of columns in the original history. + + The Histories must have same indexes, otherwise a `ValueError` is raised. + + Parameters + ---------- + original : History + The original History + + newer : History + The newer History + + Raises + ------ + ValueError : if indexes of histories does not match. + + Returns + ------- + History with appended columns + """ + original = original.copy() + + if not original.index.equals(newer.index): + raise ValueError("Index of histories does not match") + + n = len(original.columns) + append_hist = newer.hist.iloc[:, n:] + append_mask = newer.mask.iloc[:, n:] + original.hist.loc[:, append_hist.columns] = append_hist + original.mask.loc[:, append_mask.columns] = append_mask + + assert original.columns.equals(pd.Index(range(len(original.columns)))) + return original + + diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py index f69b12bb8681b71160cd4b633421d977da227c3b..33518f96a9ca22b5d6f933a004ded3a799e76c5c 100644 --- a/saqc/funcs/resampling.py +++ b/saqc/funcs/resampling.py @@ -13,12 +13,12 @@ from dios import DictOfSeries from saqc.constants import * from saqc.core.register import register, isflagged -from saqc.flagger import Flagger, initFlagsLike, History +from saqc.flagger.history import appendNewerHistory +from saqc.flagger.flags import Flagger, applyFunctionOnHistory from saqc.funcs.tools import copy, drop, rename from saqc.funcs.interpolation import interpolateIndex, _SUPPORTED_METHODS from saqc.lib.tools import evalFreqStr, getFreqDelta from saqc.lib.ts_operators import shift2Freq, aggregate2Freq -from saqc.flagger.flags import applyFunctionOnHistory, appendHistory from saqc.lib.rolling import customRoller logger = logging.getLogger("SaQC") @@ -713,5 +713,6 @@ def reindexFlags( raise ValueError(f"unknown method {method}") tmp_flagger = applyFunctionOnHistory(flagger, source, func, func_kws, func, mask_kws, last_column=dummy) - flagger = appendHistory(flagger, field, tmp_flagger.history[source]) + new_hist = appendNewerHistory(flagger.history[field], tmp_flagger.history[source]) + flagger.history[field] = new_hist return data, flagger