fixed History append

saqc/flagger/flags.py

@@ -356,18 +356,33 @@ def applyFunctionOnHistory(
 
     Parameters
     ----------
-    flags :
-    column :
-    hist_func :
-    hist_kws :
-    mask_func :
-    mask_kws :
-    last_column :
-    func_handle_df :
+    flags : Flags
+        Flags object holding the History in question
+    column : str
+        name of the column holding the history in question
+    hist_func : callable
+        function to apply on `History.hist` (flags)
+    hist_kws : dict
+        hist-function keywords dict
+    mask_func : callable
+        function to apply on `History.mask` (force mask)
+    mask_kws : dict
+        mask-function keywords dict
+    last_column : pd.Series or None, default None
+        The last column to apply. If None, no extra column is appended.
+    func_handle_df : bool
+        If `True`, the whole History{.hist, .mask} are passed to the given functions, thus the
+        function must handle `pd.Dataframes` as first input. If `False`, each column is passed
+        separately, thus the functions must handle those.
+
+    Notes
+    -----
+    After the functions are called, all `NaN`'s in `History.mask` are replaced with `False`,
+    and the `.mask` is casted to bool, to ensure a consistent History.
 
     Returns
     -------
-
+    Copy of Flags with altered History (in column)
     """
     flags = flags.copy()
     history = flags.history[column]
@@ -398,32 +413,5 @@ def applyFunctionOnHistory(
     return flags
 
 
-def appendHistory(flags: Flags, column, append_hist):
-    """
-    Function, specialized for used in deharm context.
-
-
-    Parameters
-    ----------
-    flags
-    field
-    source
-    merge_func
-    merge_func_kws
-    last_column
-
-    Returns
-    -------
-
-    """
-    flags = flags.copy()
-    new_history = flags.history[column]
-    for app_k in [k for k in append_hist.columns if k not in new_history.columns]:
-        new_history.hist[app_k] = append_hist.hist[app_k]
-        new_history.mask[app_k] = append_hist.mask[app_k]
-    flags.history[column] = new_history
-    return flags
-
-
 # for now we keep this name
 Flagger = Flags

saqc/flagger/history.py

@@ -393,3 +393,45 @@ class History:
             raise ValueError('dtype must be float')
 
         return obj
+
+
+def appendNewerHistory(original: History, newer: History) -> History:
+    """
+    Append all newer columns of a history to an other History.
+
+    The first N columns in the newer History are discarded, where N is the
+    number of columns in the original history.
+
+    The Histories must have same indexes, otherwise a `ValueError` is raised.
+
+    Parameters
+    ----------
+    original : History
+        The original History
+
+    newer : History
+        The newer History
+
+    Raises
+    ------
+    ValueError : if indexes of histories does not match.
+
+    Returns
+    -------
+    History with appended columns
+    """
+    original = original.copy()
+
+    if not original.index.equals(newer.index):
+        raise ValueError("Index of histories does not match")
+
+    n = len(original.columns)
+    append_hist = newer.hist.iloc[:, n:]
+    append_mask = newer.mask.iloc[:, n:]
+    original.hist.loc[:, append_hist.columns] = append_hist
+    original.mask.loc[:, append_mask.columns] = append_mask
+
+    assert original.columns.equals(pd.Index(range(len(original.columns))))
+    return original
+
+

saqc/funcs/resampling.py

@@ -13,12 +13,12 @@ from dios import DictOfSeries
 
 from saqc.constants import *
 from saqc.core.register import register, isflagged
-from saqc.flagger import Flagger, initFlagsLike, History
+from saqc.flagger.history import appendNewerHistory
+from saqc.flagger.flags import Flagger, applyFunctionOnHistory
 from saqc.funcs.tools import copy, drop, rename
 from saqc.funcs.interpolation import interpolateIndex, _SUPPORTED_METHODS
 from saqc.lib.tools import evalFreqStr, getFreqDelta
 from saqc.lib.ts_operators import shift2Freq, aggregate2Freq
-from saqc.flagger.flags import applyFunctionOnHistory, appendHistory
 from saqc.lib.rolling import customRoller
 
 logger = logging.getLogger("SaQC")
@@ -713,5 +713,6 @@ def reindexFlags(
         raise ValueError(f"unknown method {method}")
 
     tmp_flagger = applyFunctionOnHistory(flagger, source, func, func_kws, func, mask_kws, last_column=dummy)
-    flagger = appendHistory(flagger, field, tmp_flagger.history[source])
+    new_hist = appendNewerHistory(flagger.history[field], tmp_flagger.history[source])
+    flagger.history[field] = new_hist
     return data, flagger