From c498f0b4f125832606ca6a1cca71d90c484598a3 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 26 Feb 2021 14:34:45 +0100
Subject: [PATCH 001/180] init core adjusted
---
saqc/core/core.py | 62 ++++++++++++++++++--------------------
saqc/flagger/__init__.py | 2 ++
saqc/flagger/flags.py | 4 +++
test/core/test_core_new.py | 20 ++++++++++++
4 files changed, 55 insertions(+), 33 deletions(-)
create mode 100644 test/core/test_core_new.py
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 8a8b6283c..df90d80a0 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -10,7 +10,7 @@ TODOS:
import logging
import copy as stdcopy
-from typing import List, Tuple, Sequence
+from typing import List, Tuple, Sequence, Union
from typing_extensions import Literal
import pandas as pd
@@ -19,7 +19,8 @@ import numpy as np
import timeit
import inspect
-from saqc.flagger import BaseFlagger, CategoricalFlagger, SimpleFlagger, DmpFlagger
+from saqc.common import *
+from saqc.flagger.flags import init_flags_like, Flagger
from saqc.core.lib import APIController, ColumnSelector
from saqc.core.register import FUNC_MAP, SaQCFunction
from saqc.core.modules import FuncModules
@@ -49,7 +50,8 @@ def _handleErrors(exc: Exception, field: str, control: APIController, func: SaQC
raise exc
-def _prepInput(flagger, data, flags):
+# todo: shouldt this go to Saqc.__init__ ?
+def _prepInput(data, flags):
dios_like = (dios.DictOfSeries, pd.DataFrame)
if isinstance(data, pd.Series):
@@ -66,30 +68,23 @@ def _prepInput(flagger, data, flags):
if not hasattr(data.columns, "str"):
raise TypeError("expected dataframe columns of type string")
- if not isinstance(flagger, BaseFlagger):
- # NOTE: we should generate that list automatically,
- # it won't ever be complete otherwise
- flaggerlist = [CategoricalFlagger, SimpleFlagger, DmpFlagger]
- raise TypeError(f"'flagger' must be of type {flaggerlist} or a subclass of {BaseFlagger}")
-
if flags is not None:
- if not isinstance(flags, dios_like):
- raise TypeError("'flags' must be of type dios.DictOfSeries or pd.DataFrame")
if isinstance(flags, pd.DataFrame):
if isinstance(flags.index, pd.MultiIndex) or isinstance(flags.columns, pd.MultiIndex):
raise TypeError("'flags' should not use MultiIndex")
- flags = dios.to_dios(flags)
- # NOTE: do not test all columns as they not necessarily need to be the same
- cols = flags.columns & data.columns
- if not (flags[cols].lengths == data[cols].lengths).all():
- raise ValueError("the length of 'flags' and 'data' need to be equal")
+ if isinstance(flags, (dios.DictOfSeries, pd.DataFrame, Flagger)):
+ # NOTE: only test common columns, data as well as flags could
+ # have more columns than the respective other.
+ cols = flags.columns & data.columns
+ for c in cols:
+ if not flags[c].index.equals(data[c].index):
+ raise ValueError(f"the index of 'flags' and 'data' missmatch in column {c}")
- if flagger.initialized:
- diff = data.columns.difference(flagger.getFlags().columns)
- if not diff.empty:
- raise ValueError("Missing columns in 'flagger': '{list(diff)}'")
+ # this also ensures float dtype
+ if not isinstance(flags, Flagger):
+ flags = Flagger(flags, copy=True)
return data, flags
@@ -110,31 +105,32 @@ _setup()
class SaQC(FuncModules):
- def __init__(self, flagger, data, flags=None, nodata=np.nan, to_mask=None, error_policy="raise"):
+ def __init__(self, data, flags=None, nodata=np.nan, to_mask=None, error_policy="raise"):
super().__init__(self)
- data, flags = _prepInput(flagger, data, flags)
+ data, flagger = _prepInput(data, flags)
self._data = data
self._nodata = nodata
self._to_mask = to_mask
- self._flagger = self._initFlagger(data, flagger, flags)
+ self._flagger = self._initFlagger(data, flags)
self._error_policy = error_policy
# NOTE: will be filled by calls to `_wrap`
self._to_call: List[Tuple[ColumnSelector, APIController, SaQCFunction]] = []
- def _initFlagger(self, data, flagger, flags):
+ def _initFlagger(self, data, flagger: Union[Flagger, None]):
""" Init the internal flagger object.
Ensures that all data columns are present and user passed flags from
- a flags frame and/or an already initialised flagger are used.
- If columns overlap the passed flagger object is prioritised.
+ a flags frame or an already initialised flagger are used.
"""
- # ensure all data columns
- merged = flagger.initFlags(data)
- if flags is not None:
- merged = merged.merge(flagger.initFlags(flags=flags), inplace=True)
- if flagger.initialized:
- merged = merged.merge(flagger, inplace=True)
- return merged
+ if flagger is None:
+ return init_flags_like(data)
+
+ for c in flagger.columns.union(data.columns):
+ if c in flagger:
+ continue
+ if c in data:
+ flagger[c] = pd.Series(UNFLAGGED, index=data[c].index, dtype=float)
+ return flagger
def readConfig(self, fname):
from saqc.core.reader import readConfig
diff --git a/saqc/flagger/__init__.py b/saqc/flagger/__init__.py
index d5124fb9d..774f2ec2b 100644
--- a/saqc/flagger/__init__.py
+++ b/saqc/flagger/__init__.py
@@ -1,6 +1,8 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
+from .flags import Flagger
+from .history import History
from saqc.flagger.baseflagger import BaseFlagger
from saqc.flagger.categoricalflagger import CategoricalFlagger
from saqc.flagger.simpleflagger import SimpleFlagger
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index bf64ec556..15b8a4efc 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -311,3 +311,7 @@ def init_flags_like(reference: Union[pd.Series, DictLike, Flags], initial_value:
return Flags(result)
+
+# for now we keep this name
+Flagger = Flags
+
diff --git a/test/core/test_core_new.py b/test/core/test_core_new.py
new file mode 100644
index 000000000..b16714c8e
--- /dev/null
+++ b/test/core/test_core_new.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+import pandas as pd
+import numpy as np
+import dios
+
+
+def test_init():
+ from saqc import SaQC, Flagger
+
+ arr = np.array([
+ [0, 1, 2],
+ [0, 1, 3],
+ ])
+ data = pd.DataFrame(arr, columns=list('abc'))
+ qc = SaQC(data)
+
+ assert isinstance(qc, SaQC)
+ assert isinstance(qc._flagger, Flagger)
+ assert isinstance(qc._data, dios.DictOfSeries)
--
GitLab
From 4aa3d4ea55c8ba3b64fa703998c2c62b511098c5 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 26 Feb 2021 14:55:19 +0100
Subject: [PATCH 002/180] rm AnyFlagger, refactored BaseFlagger -> Flagger
---
saqc/core/core.py | 2 +-
saqc/core/modules/changepoints.py | 11 +-
saqc/flagger/__init__.py | 7 -
saqc/flagger/baseflagger.py | 481 -----------------------------
saqc/flagger/categoricalflagger.py | 40 ---
saqc/flagger/compatflagger.py | 481 -----------------------------
saqc/flagger/continuousflagger.py | 47 ---
saqc/flagger/dmpflagger.py | 248 ---------------
saqc/flagger/positionalflagger.py | 144 ---------
saqc/flagger/simpleflagger.py | 12 -
saqc/funcs/breaks.py | 16 +-
saqc/funcs/changepoints.py | 10 +-
saqc/funcs/constants.py | 16 +-
saqc/funcs/curvefit.py | 10 +-
saqc/funcs/drift.py | 46 +--
saqc/funcs/flagtools.py | 34 +-
saqc/funcs/generic.py | 18 +-
saqc/funcs/interpolation.py | 26 +-
saqc/funcs/outliers.py | 68 ++--
saqc/funcs/pattern.py | 16 +-
saqc/funcs/resampling.py | 62 ++--
saqc/funcs/residues.py | 14 +-
saqc/funcs/rolling.py | 8 +-
saqc/funcs/scores.py | 6 +-
saqc/funcs/tools.py | 28 +-
saqc/funcs/transformation.py | 10 +-
saqc/lib/plotting.py | 20 +-
27 files changed, 208 insertions(+), 1673 deletions(-)
delete mode 100644 saqc/flagger/baseflagger.py
delete mode 100644 saqc/flagger/categoricalflagger.py
delete mode 100644 saqc/flagger/compatflagger.py
delete mode 100644 saqc/flagger/continuousflagger.py
delete mode 100644 saqc/flagger/dmpflagger.py
delete mode 100644 saqc/flagger/positionalflagger.py
delete mode 100644 saqc/flagger/simpleflagger.py
diff --git a/saqc/core/core.py b/saqc/core/core.py
index df90d80a0..0dae725a3 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -349,7 +349,7 @@ def _warnForUnusedKwargs(func, flagger):
----------
func: SaqcFunction
Saqc internal data structure that hold all function info.
- flagger: saqc.flagger.BaseFlagger
+ flagger: saqc.flagger.Flagger
Flagger object.
Returns
diff --git a/saqc/core/modules/changepoints.py b/saqc/core/modules/changepoints.py
index 0586a3ac8..58093bc17 100644
--- a/saqc/core/modules/changepoints.py
+++ b/saqc/core/modules/changepoints.py
@@ -1,15 +1,10 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from typing import Callable, Union, Tuple
+from typing import Callable, Union
from typing_extensions import Literal
-
import numpy as np
-
-from dios import DictOfSeries
-
from saqc.core.modules.base import ModuleBase
-from saqc.flagger.baseflagger import BaseFlagger
class ChangePoints(ModuleBase):
@@ -28,7 +23,7 @@ class ChangePoints(ModuleBase):
reduce_window: str = None,
reduce_func: Callable[[np.array, np.array], np.array] = lambda x, y: x.argmax(),
**kwargs
- ) -> Tuple[DictOfSeries, BaseFlagger]:
+ ):
return self.defer("flagChangePoints", locals())
@@ -49,6 +44,6 @@ class ChangePoints(ModuleBase):
flag_changepoints: bool = False,
assign_cluster: bool = True,
**kwargs
- ) -> Tuple[DictOfSeries, BaseFlagger]:
+ ):
return self.defer("assignChangePointCluster", locals())
diff --git a/saqc/flagger/__init__.py b/saqc/flagger/__init__.py
index 774f2ec2b..890db7ee1 100644
--- a/saqc/flagger/__init__.py
+++ b/saqc/flagger/__init__.py
@@ -3,10 +3,3 @@
from .flags import Flagger
from .history import History
-from saqc.flagger.baseflagger import BaseFlagger
-from saqc.flagger.categoricalflagger import CategoricalFlagger
-from saqc.flagger.simpleflagger import SimpleFlagger
-from saqc.flagger.dmpflagger import DmpFlagger
-from saqc.flagger.continuousflagger import ContinuousFlagger
-from saqc.flagger.positionalflagger import PositionalFlagger
-from saqc.flagger.compatflagger import CompatFlagger
diff --git a/saqc/flagger/baseflagger.py b/saqc/flagger/baseflagger.py
deleted file mode 100644
index 991e5751a..000000000
--- a/saqc/flagger/baseflagger.py
+++ /dev/null
@@ -1,481 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-from __future__ import annotations
-
-import operator as op
-from copy import deepcopy
-from abc import ABC, abstractmethod
-
-from typing import TypeVar, Union, Any, List, Optional, Tuple, Dict, Sequence
-
-import pandas as pd
-import numpy as np
-
-from dios import DictOfSeries
-
-from saqc.lib.tools import assertScalar, mergeDios, toSequence, customRoller
-
-
-COMPARATOR_MAP = {
- "!=": op.ne,
- "==": op.eq,
- ">=": op.ge,
- ">": op.gt,
- "<=": op.le,
- "<": op.lt,
-}
-
-
-LocT = Union[pd.Series, pd.Index, slice]
-FlagT = Any
-BaseFlaggerT = TypeVar("BaseFlaggerT", bound="BaseFlagger")
-PandasT = Union[pd.Series, DictOfSeries]
-FieldsT = Union[str, List[str], slice]
-
-
-class BaseFlagger(ABC):
- @abstractmethod
- def __init__(self, dtype):
- # NOTE: the type of the _flags DictOfSeries
- self.dtype = dtype
- self.extra_defaults = dict()
- # NOTE: the arggumens of setFlags supported from
- # the configuration functions
- self.signature = ("flag", "force", "flag_after", "flag_before")
- self._flags: Optional[DictOfSeries] = None
-
- @property
- def initialized(self):
- return self._flags is not None
-
- @property
- def flags(self):
- return self._flags.copy()
-
- def initFlags(self, data: DictOfSeries = None, flags: DictOfSeries = None) -> BaseFlagger:
- """
- initialize a flagger based on the given 'data' or 'flags'
- if 'data' is not None: return a flagger with flagger.UNFLAGGED values
- if 'flags' is not None: return a flagger with the given flags
- """
-
- if data is None and flags is None:
- raise TypeError("either 'data' or 'flags' are required")
-
- if data is not None and flags is not None:
- raise TypeError("either 'data' or 'flags' can be given")
-
- if data is not None:
- if not isinstance(data, DictOfSeries):
- data = DictOfSeries(data)
-
- flags = DictOfSeries(columns=data.columns)
- for c in flags.columns.tolist():
- flags[c] = pd.Series(self.UNFLAGGED, index=data[c].index)
- else:
- if not isinstance(flags, DictOfSeries):
- flags = DictOfSeries(flags)
-
- flags = flags.astype(self.dtype)
- newflagger = self.copy(flags=flags)
- return newflagger
-
- def merge(self, other: BaseFlagger, subset: Optional[List] = None, join: str = "merge", inplace=False) -> BaseFlagger:
- """
- Merge the given flagger 'other' into self
- """
- # NOTE: add more checks !?
- if not isinstance(other, self.__class__):
- raise TypeError(f"flagger of type '{self.__class__}' needed")
-
- if inplace:
- self._flags = mergeDios(self._flags, other._flags, subset=subset, join=join)
- return self
- else:
- return self.copy(flags=mergeDios(self._flags, other._flags, subset=subset, join=join))
-
- def slice(self, field: FieldsT = None, loc: LocT = None, drop: FieldsT = None, inplace=False) -> BaseFlagger:
- """ Return a potentially trimmed down copy of self. """
- if drop is not None:
- if field is not None:
- raise TypeError("either 'field' or 'drop' can be given, but not both")
- field = self._flags.columns.drop(drop, errors="ignore")
- flags = self.getFlags(field=field, loc=loc).to_dios() # type: ignore
-
- if inplace:
- self._flags = flags
- return self
- else:
- return self.copy(flags=flags)
-
- def toFrame(self):
- """ Return a pd.DataFrame holding the flags
- Return
- ------
- frame: pandas.DataFrame
-
- Note
- ----
- This is a convenience funtion hiding the implementation detail dios.DictOfSeries.
- Subclasses with special flag structures (i.e. DmpFlagger) should overwrite the
- this methode in order to provide a usefull user output.
- """
- return self._flags.to_df()
-
- def getFlags(self, field: FieldsT = None, loc: LocT = None, full=False) -> Union[PandasT, Tuple[DictOfSeries, Dict[str, PandasT]]]:
- """ Return a potentially, to `loc`, trimmed down version of flags.
-
- Parameters
- ----------
- field : str, list of str or None, default None
- Field(s) to request.
- loc :
- limit result to specific rows.
- full : object
- If True, an additional dict is returned, holding all extras that
- the flagger may specify. These extras can be feed back to a/the
- flagger with `setFlags(...with_extras=True)`.
-
- Return
- ------
- flags: pandas.Series or dios.DictOfSeries
- If field is a scalar a series is returned, otherwise a dios.
- extras: dict
- Present only if `full=True`. A dict that hold all extra information.
-
- Note
- ----
- This is more or less a __getitem__(key)-like function, where
- self._flags is accessed and key is a single key or a tuple.
- Either key is [loc] or [loc,field]. loc also can be a 2D-key,
- aka. a booldios
-
- The resulting dict (full=True) can be feed to setFlags to update extra Columns.
- but field must be a scalar then, because setFlags only can process a scalar field.
- """
-
- # loc should be a valid 2D-indexer and
- # then field must be None. Otherwise aloc
- # will fail and throw the correct Error.
- if isinstance(loc, DictOfSeries) and field is None:
- indexer = loc
- else:
- rows = slice(None) if loc is None else loc
- cols = slice(None) if field is None else self._check_field(field)
- indexer = (rows, cols)
-
- # this is a bug in `dios.aloc`, which may return a shallow copied dios, if `slice(None)` is passed
- # as row indexer. Thus is because pandas `.loc` return a shallow copy if a null-slice is passed to a series.
- flags = self._flags.aloc[indexer].copy()
- if full:
- return flags, {}
- else:
- return flags
-
- def setFlags(
- self,
- field: str,
- loc: LocT = None,
- flag: FlagT = None,
- force: bool = False,
- inplace: bool = False,
- with_extra: bool = False,
- flag_after: Union[str, int] = None,
- flag_before: Union[str, int] = None,
- win_flag: FlagT = None,
- **kwargs
- ) -> BaseFlagger:
- """Overwrite existing flags at loc.
-
- If `force=False` (default) only flags with a lower priority are overwritten,
- otherwise, if `force=True`, flags are overwritten unconditionally.
-
- Examples
- --------
- One can use this to update extra columns without knowing their names. Eg. like so:
-
- >>> field = 'var0'
- >>> flags, extra = flagger.getFlags(field, full=True)
- >>> newflags = magic_that_alter_index(flags)
- >>> for k, v in extra.items()
- ... extra[k] = magic_that_alter_index(v)
- >>> flagger = flagger.setFlags(field, flags=newflags, with_extra=True, **extra)
- """
-
- assert "iloc" not in kwargs, "deprecated keyword, `iloc=slice(i:j)`. Use eg. `loc=srs.index[i:j]` instead."
-
- assertScalar("field", self._check_field(field), optional=False)
- flag = self.BAD if flag is None else flag
- out = self if inplace else deepcopy(self)
-
- if with_extra and not isinstance(flag, pd.Series):
- raise ValueError("flags must be pd.Series if `with_extras=True`.")
-
- trimmed = self.getFlags(field=field, loc=loc)
- if force:
- mask = pd.Series(True, index=trimmed.index, dtype=bool) # type: ignore / `trimmed` is asserted to be a Series
- else:
- mask = trimmed < flag
-
- # set flags of the test
- out._flags.aloc[mask, field] = flag
-
- # calc and set window flags
- if flag_after is not None or flag_before is not None:
- win_mask, win_flag = self._getWindowMask(field, mask, flag_after, flag_before, win_flag, flag, force)
- out._flags.aloc[win_mask, field] = win_flag
-
- return out
-
- def _getWindowMask(self, field, mask, flag_after, flag_before, win_flag, flag, force):
- """ Return a mask which is True where the additional window flags should get set.
-
- Parameters
- ----------
- field : str
- column identifier.
- mask : boolean pd.Series
- identified locations where flags was set
- flag_after : offset or int
- set additional flags after each flag that was set
- flag_before : offset or int
- set additional flags before each flag that was set
- win_flag : any
- Should be valid flag of the flagger or None. Defaults to `flag` if None.
- flag : any
- The flag that was used by flagger.setFlags(). Only used to determine `win_flag` if the latter is None.
- force : bool
- If True, the additional flags specified by `flag_after` and `flag_before` are set unconditionally and so
- also could overwrite worse flags.
-
- Returns
- -------
- mask: boolean pandas.Series
- locations where additional flags should be set. The mask has the same (complete) length than `.flags[field]`
- win_flag: the flag to set
-
- Raises
- ------
- ValueError : If `win_flag` is None and `flag` is not a scalar.
- ValueError : If `win_flag` is not a valid flagger flag
- NotImplementedError: if `flag_before` is given
- """
-
- # win_flag default to flag if not explicitly given
- if win_flag is None:
- win_flag = flag
- if not np.isscalar(win_flag):
- raise ValueError("win_flag (None) cannot default to flag, if flag is not a scalar. "
- "Pls specify `win_flag` or omit `flag_after` and `flag_before`.")
- else:
- if not self.isValidFlag(win_flag):
- raise ValueError(f"invalid win_flag: {win_flag}")
-
- # blow up the mask to the whole size of flags
- base = mask.reindex_like(self._flags[field]).fillna(False)
- before, after = False, False
-
- if flag_before is not None:
- closed = 'both'
- if isinstance(flag_before, int):
- flag_before, closed = flag_before + 1, None
- r = customRoller(base, window=flag_before, min_periods=1, closed=closed, expand=True, forward=True)
- before = r.sum().astype(bool)
-
- if flag_after is not None:
- closed = 'both'
- if isinstance(flag_after, int):
- flag_after, closed = flag_after + 1, None
- r = customRoller(base, window=flag_after, min_periods=1, closed=closed, expand=True)
- after = r.sum().astype(bool)
-
- # does not include base, to avoid overriding flags that just was set
- # by the test, because flag and win_flag may differ.
- mask = ~base & (after | before)
-
- # also do not to overwrite worse flags
- if not force:
- mask &= self.getFlags(field) < win_flag
-
- return mask, win_flag
-
- def clearFlags(self, field: str, loc: LocT = None, inplace: bool=False, **kwargs) -> BaseFlagger:
- assertScalar("field", field, optional=False)
- if "force" in kwargs:
- raise ValueError("Keyword 'force' is not allowed here.")
- if "flag" in kwargs:
- raise ValueError("Keyword 'flag' is not allowed here.")
- return self.setFlags(field=field, loc=loc, flag=self.UNFLAGGED, force=True, inplace=inplace, **kwargs)
-
- def isFlagged(self, field=None, loc: LocT = None, flag: FlagT = None, comparator: str=">") -> PandasT:
- """
- Returns boolean data that indicate where data has been flagged.
-
- Parameters
- ----------
- field : str, list-like, default None
- The field(s)/column(s) of the data to be tested if flagged.
- If None all columns are used.
-
- loc : mask, slice, pd.Index, etc., default None
- The location/rows of the data to be tested if flagged.
- If None all rows are used.
-
- flag : str, category, list-like, default None
- The flag(s) that define data as flagged. If None, `flagger.GOOD`
- is used.
-
- comparator : {'<', '<=', '==', '!=', '>=', '>'}, default '>'
- Defines how the comparison is done. The `flags` are always on the
- left-hand-side, thus, the flag to compare is always on the right-
- hand-side. Eg. a call with all defaults, return the equivalent
- of `flagger.getFlags() > flagger.GOOD`
-
- Returns
- -------
- pandas.Series or dios.DictOfSeries : Return Series if field is a scalar,
- otherwise DictOfSeries.
- """
- if isinstance(flag, pd.Series):
- raise TypeError("flag: pd.Series is not allowed")
- flags_to_compare = set(toSequence(flag, self.GOOD))
-
- flags = self.getFlags(field, loc, full=False)
- cp = COMPARATOR_MAP[comparator]
-
- # notna() to prevent nans to become True,
- # eg.: `np.nan != 0 -> True`
- flagged = flags.notna() # type: ignore / we asserted, that flags is of `PandasT`
-
- # passing an empty list must result
- # in a everywhere-False data
- if len(flags_to_compare) == 0:
- flagged[:] = False
- else:
- for f in flags_to_compare:
- if not self.isValidFlag(f):
- raise ValueError(f"invalid flag: {f}")
- flagged &= cp(flags, f)
-
- return flagged
-
- def copy(self, flags: Optional[PandasT]=None) -> BaseFlagger:
- if flags is None:
- out = deepcopy(self)
- else:
- # if flags is given and self.flags is big,
- # this hack will bring some speed improvement
- # NOTE: there should be nicer way to do this,
- # why not through a constructur method?
- saved = self._flags
- self._flags = None
- out = deepcopy(self)
- out._flags = flags.copy()
- self._flags = saved
- return out
-
- def isValidFlag(self, flag: FlagT) -> bool:
- """
- Check if given flag is known to this flagger.
-
- Parameters
- ----------
- flag: str
- The flag to be checked.
-
- Returns
- -------
- bool
- """
- # This is a very rudimentary fallback for the check
- # and the child flagger may should implement a better
- # version of it.
- return flag == self.BAD or flag == self.GOOD or flag == self.UNFLAGGED or self.isSUSPICIOUS(flag)
-
- def replaceField(self, field: str, flags: Optional[pd.Series], inplace: bool=False, **kwargs) -> BaseFlagger:
- """ Replace or delete all data for a given field.
-
- Parameters
- ----------
- field : str
- The field to replace / delete. If the field already exist, the respected data
- is replaced, otherwise the data is inserted in the respected field column.
- flags : pandas.Series or None
- If None, the series denoted by `field` will be deleted. Otherwise
- a series of flags (dtype flagger.dtype) that will replace the series
- currently stored under `field`
- inplace : bool, default False
- If False, a flagger copy is returned, otherwise the flagger is not copied.
- **kwargs : dict
- ignored.
-
- Returns
- -------
- flagger: saqc.flagger.BaseFlagger
- The flagger object or a copy of it (if inplace=True).
-
- Raises
- ------
- ValueError: (delete) if field does not exist
- TypeError: (replace / insert) if flags are not pd.Series
- """
-
- assertScalar("field", field, optional=False)
-
- out = self if inplace else deepcopy(self)
-
- # delete
- if flags is None:
- if field not in self._flags:
- raise ValueError(f"{field}: field does not exist")
- del out._flags[field]
-
- # insert / replace
- else:
- if not isinstance(flags, pd.Series):
- raise TypeError(f"`flags` must be pd.Series.")
- out._flags[field] = flags.astype(self.dtype)
- return out
-
- def _check_field(self, field: Union[str, Sequence[str]]) -> Union[str, Sequence[str]]:
- """ Check if (all) field(s) in self._flags. """
-
- # wait for outcome of
- # https://git.ufz.de/rdm-software/saqc/issues/46
- failed = []
- if isinstance(field, str):
- if field not in self._flags:
- failed += [field]
- else:
- try:
- for f in field:
- if f not in self._flags:
- failed += [f]
- # not iterable, probably a slice or
- # any indexer we dont have to check
- except TypeError:
- pass
-
- if failed:
- raise ValueError(f"key(s) missing in flags: {failed}")
- return field
-
- @property
- @abstractmethod
- def UNFLAGGED(self) -> FlagT:
- """ Return the flag that indicates unflagged data """
-
- @property
- @abstractmethod
- def GOOD(self) -> FlagT:
- """ Return the flag that indicates the very best data """
-
- @property
- @abstractmethod
- def BAD(self) -> FlagT:
- """ Return the flag that indicates the worst data """
-
- @abstractmethod
- def isSUSPICIOUS(self, flag: FlagT) -> bool:
- """ Return bool that indicates if the given flag is valid, but neither
- UNFLAGGED, BAD, nor GOOD."""
diff --git a/saqc/flagger/categoricalflagger.py b/saqc/flagger/categoricalflagger.py
deleted file mode 100644
index 598cdb868..000000000
--- a/saqc/flagger/categoricalflagger.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from collections import OrderedDict
-
-import pandas as pd
-
-from saqc.flagger.baseflagger import BaseFlagger
-
-
-class Flags(pd.CategoricalDtype):
- def __init__(self, flags):
- # NOTE: all flag schemes need to support
- # at least 3 flag categories:
- # * unflagged
- # * good
- # * bad
- assert len(flags) > 2
- super().__init__(flags, ordered=True)
-
-
-class CategoricalFlagger(BaseFlagger):
- def __init__(self, flags):
- super().__init__(dtype=Flags(flags))
- self._categories = self.dtype.categories
-
- @property
- def UNFLAGGED(self):
- return self._categories[0]
-
- @property
- def GOOD(self):
- return self._categories[1]
-
- @property
- def BAD(self):
- return self._categories[-1]
-
- def isSUSPICIOUS(self, flag):
- return flag in self._categories[2:-1]
diff --git a/saqc/flagger/compatflagger.py b/saqc/flagger/compatflagger.py
deleted file mode 100644
index 456a2ec48..000000000
--- a/saqc/flagger/compatflagger.py
+++ /dev/null
@@ -1,481 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-from __future__ import annotations
-
-import operator as op
-from copy import deepcopy
-from typing import TypeVar, Union, Any, List, Optional, Tuple, Dict, Sequence, Type
-
-from dios import DictOfSeries
-import pandas as pd
-import numpy as np
-
-from saqc.common import *
-from saqc.lib.tools import assertScalar, mergeDios, toSequence, customRoller
-from saqc.flagger.flags import Flags, init_flags_like, History
-from saqc.flagger.baseflagger import (
- BaseFlagger,
- COMPARATOR_MAP,
- LocT,
- FlagT,
- PandasT,
- FieldsT,
-)
-
-
-class CompatFlagger(BaseFlagger):
-
- def __init__(self):
- super().__init__(float)
-
- # not supported anymore
- delattr(self, 'extra_defaults')
-
- # the arguments of setFlags supported from the configuration functions
- self.signature = ("flag", "force", "flag_after", "flag_before")
- self._flags: Optional[Flags] = None
-
- @property
- def _constructor(self) -> Type['CompatFlagger']:
- return CompatFlagger
-
- @property
- def initialized(self):
- return self._flags is not None
-
- @property
- def flags(self):
- return self._flags.to_dios()
-
- def initFlags(self, data=None, flags=None) -> CompatFlagger:
- """
- initialize a flagger based on the given 'data' or 'flags'
- if 'data' is not None: return a flagger with flagger.UNFLAGGED values
- if 'flags' is not None: return a flagger with the given flags
- """
-
- if data is None and flags is None:
- raise TypeError("either 'data' or 'flags' are required")
-
- if data is not None and flags is not None:
- raise TypeError("either 'data' or 'flags' can be given")
-
- if data is not None:
- flags = init_flags_like(data, initial_value=UNFLAGGED)
- else:
-
- try:
- flags = flags.astype(float)
- except AttributeError:
- pass
-
- flags = Flags(flags)
-
- out = self._constructor()
- out._flags = flags
- return out
-
- def merge(self, other, subset: Optional[List] = None, join: str = "merge", inplace=False) -> CompatFlagger:
- """
- Merge the given flagger 'other' into self
- """
- # NOTE: add more checks !?
- if not isinstance(other, CompatFlagger):
- raise TypeError(f"flagger of type CompatFlagger needed")
-
- raise NotImplementedError('merge not implemented')
-
- if inplace:
- self._flags = mergeDios(self._flags, other._flags, subset=subset, join=join)
- return self
- else:
- return self.copy(flags=mergeDios(self._flags, other._flags, subset=subset, join=join))
-
- def slice(self, field: FieldsT = None, loc: LocT = None, drop: FieldsT = None, inplace=False) -> CompatFlagger:
- """ Return a potentially trimmed down copy of self. """
-
- if drop is not None and field is not None:
- raise TypeError("either 'field' or 'drop' can be given, but not both")
-
- if not inplace:
- self = self.copy() # dirty
-
- to_drop = toSequence(drop, [])
- to_keep = toSequence(field, [])
-
- for c in to_drop:
- self._flags.drop(c)
-
- if to_keep:
- for c in self._flags.columns:
- if c in to_keep:
- continue
- self._flags.drop(c)
-
- if loc is not None:
- for c in self._flags.columns:
- h = self._flags.history[c].hist.loc[loc]
- m = self._flags.history[c].mask.loc[loc]
- self._flags.history[c] = History(hist=h, mask=m)
-
- return self
-
- def toFrame(self):
- """ Return a pd.DataFrame holding the flags
- Return
- ------
- frame: pandas.DataFrame
-
- Note
- ----
- This is a convenience funtion hiding the implementation detail dios.DictOfSeries.
- Subclasses with special flag structures (i.e. DmpFlagger) should overwrite the
- this methode in order to provide a usefull user output.
- """
- return self._flags.to_frame()
-
- def getFlags(self, field: FieldsT = None, loc: LocT = None, full=False) -> Union[
- PandasT, Tuple[DictOfSeries, Dict[str, PandasT]]]:
- """ Return a potentially, to `loc`, trimmed down version of flags.
-
- Parameters
- ----------
- field : str, list of str or None, default None
- Field(s) to request.
- loc :
- limit result to specific rows.
- full : object
- If True, an additional dict is returned, holding all extras that
- the flagger may specify. These extras can be feed back to a/the
- flagger with `setFlags(...with_extras=True)`.
-
- Return
- ------
- flags: pandas.Series or dios.DictOfSeries
- If field is a scalar a series is returned, otherwise a dios.
- extras: dict
- Present only if `full=True`. A dict that hold all extra information.
-
- Note
- ----
- This is more or less a __getitem__(key)-like function, where
- self._flags is accessed and key is a single key or a tuple.
- Either key is [loc] or [loc,field]. loc also can be a 2D-key,
- aka. a booldios
-
- The resulting dict (full=True) can be feed to setFlags to update extra Columns.
- but field must be a scalar then, because setFlags only can process a scalar field.
- """
-
- if full:
- raise NotImplementedError("Deprecated keyword 'full'")
-
- # loc should be a valid 2D-indexer and
- # then field must be None. Otherwise aloc
- # will fail and throw the correct Error.
- if isinstance(loc, DictOfSeries) and field is None:
- indexer = loc
- else:
- rows = slice(None) if loc is None else loc
- cols = slice(None) if field is None else self._check_field(field)
- indexer = (rows, cols)
-
- # this is a bug in `dios.aloc`, which may return a shallow copied dios, if `slice(None)` is passed
- # as row indexer. Thus is because pandas `.loc` return a shallow copy if a null-slice is passed to a series.
- flags = self.flags.aloc[indexer].copy()
- return flags
-
- def setFlags(
- self,
- field: str,
- loc: LocT = None,
- flag: FlagT = None,
- force: bool = False,
- inplace: bool = False,
- with_extra: bool = False,
- flag_after: Union[str, int] = None,
- flag_before: Union[str, int] = None,
- win_flag: FlagT = None,
- **kwargs
- ) -> CompatFlagger:
- """Overwrite existing flags at loc.
-
- If `force=False` (default) only flags with a lower priority are overwritten,
- otherwise, if `force=True`, flags are overwritten unconditionally.
- """
- if with_extra:
- raise NotImplementedError("Deprecated keyword 'with_extra'")
-
- assertScalar("field", self._check_field(field), optional=False)
- flag = BAD if flag is None else flag
- out = self if inplace else deepcopy(self)
-
- actual = self._flags[field] # a copy
-
- # new might (very possibly) have a different index
- # as the actual flags column
- new = self.getFlags(field=field, loc=loc)
- new = pd.Series(UNTOUCHED, index=new.index, dtype=float)
-
- if isinstance(flag, pd.Series):
- if flag.dtype != float:
- raise TypeError('series like flags mut be of dtype float')
- flag = flag.reindex(index=new.index)
- new[:] = flag
-
- reindexed = new.reindex(index=actual.index, fill_value=UNTOUCHED)
-
- mask = np.isfinite(reindexed)
-
- # calc and set window flags
- if flag_after is not None or flag_before is not None:
- win_mask, win_flag = out._getWindowMask(field, mask, flag_after, flag_before, win_flag, flag, force)
- reindexed[win_mask] = win_flag
-
- if force:
- out._flags.force(field, reindexed)
- else:
- out._flags[field] = reindexed
-
- return out
-
- def _getWindowMask(self, field, mask, flag_after, flag_before, win_flag, flag, force):
- """ Return a mask which is True where the additional window flags should get set.
-
- Parameters
- ----------
- field : str
- column identifier.
- mask : boolean pd.Series
- identified locations where flags was set
- flag_after : offset or int
- set additional flags after each flag that was set
- flag_before : offset or int
- set additional flags before each flag that was set
- win_flag : any
- Should be valid flag of the flagger or None. Defaults to `flag` if None.
- flag : any
- The flag that was used by flagger.setFlags(). Only used to determine `win_flag` if the latter is None.
- force : bool
- If True, the additional flags specified by `flag_after` and `flag_before` are set unconditionally and so
- also could overwrite worse flags.
-
- Returns
- -------
- mask: boolean pandas.Series
- locations where additional flags should be set. The mask has the same (complete) length than `.flags[field]`
- win_flag: the flag to set
-
- Raises
- ------
- ValueError : If `win_flag` is None and `flag` is not a scalar.
- ValueError : If `win_flag` is not a valid flagger flag
- NotImplementedError: if `flag_before` is given
- """
-
- # win_flag default to flag if not explicitly given
- if win_flag is None:
- win_flag = flag
- if not np.isscalar(win_flag):
- raise ValueError("win_flag (None) cannot default to flag, if flag is not a scalar. "
- "Pls specify `win_flag` or omit `flag_after` and `flag_before`.")
- else:
- if not self.isValidFlag(win_flag):
- raise ValueError(f"invalid win_flag: {win_flag}")
-
- # blow up the mask to the whole size of flags
- base = mask.reindex_like(self.flags[field]).fillna(False)
- before, after = False, False
-
- if flag_before is not None:
- closed = 'both'
- if isinstance(flag_before, int):
- flag_before, closed = flag_before + 1, None
- r = customRoller(base, window=flag_before, min_periods=1, closed=closed, expand=True, forward=True)
- before = r.sum().astype(bool)
-
- if flag_after is not None:
- closed = 'both'
- if isinstance(flag_after, int):
- flag_after, closed = flag_after + 1, None
- r = customRoller(base, window=flag_after, min_periods=1, closed=closed, expand=True)
- after = r.sum().astype(bool)
-
- # does not include base, to avoid overriding flags that just was set
- # by the test, because flag and win_flag may differ.
- mask = ~base & (after | before)
-
- # also do not to overwrite worse flags
- if not force:
- mask &= self.getFlags(field) < win_flag
-
- return mask, win_flag
-
- def clearFlags(self, field: str, loc: LocT = None, inplace: bool = False, **kwargs) -> CompatFlagger:
- assertScalar("field", field, optional=False)
- if "force" in kwargs:
- raise ValueError("Keyword 'force' is not allowed here.")
- if "flag" in kwargs:
- raise ValueError("Keyword 'flag' is not allowed here.")
- return self.setFlags(field=field, loc=loc, flag=UNFLAGGED, force=True, inplace=inplace, **kwargs)
-
- def isFlagged(self, field=None, loc: LocT = None, flag: FlagT = None, comparator: str = ">") -> PandasT:
- """
- Returns boolean data that indicate where data has been flagged.
-
- Parameters
- ----------
- field : str, list-like, default None
- The field(s)/column(s) of the data to be tested if flagged.
- If None all columns are used.
-
- loc : mask, slice, pd.Index, etc., default None
- The location/rows of the data to be tested if flagged.
- If None all rows are used.
-
- flag : str, category, list-like, default None
- The flag(s) that define data as flagged. If None, `flagger.GOOD`
- is used.
-
- comparator : {'<', '<=', '==', '!=', '>=', '>'}, default '>'
- Defines how the comparison is done. The `flags` are always on the
- left-hand-side, thus, the flag to compare is always on the right-
- hand-side. Eg. a call with all defaults, return the equivalent
- of `flagger.getFlags() > flagger.GOOD`
-
- Returns
- -------
- pandas.Series or dios.DictOfSeries : Return Series if field is a scalar,
- otherwise DictOfSeries.
- """
- if isinstance(flag, pd.Series):
- raise TypeError("flag: pd.Series is not allowed")
- flags_to_compare = set(toSequence(flag, UNFLAGGED))
-
- flags = self.getFlags(field, loc, full=False)
- cp = COMPARATOR_MAP[comparator]
-
- # notna() to prevent nans to become True,
- # eg.: `np.nan != 0 -> True`
- flagged = flags.notna() # _type: ignore / we asserted, that flags is of `PandasT`
-
- # passing an empty list must result
- # in a everywhere-False data
- if len(flags_to_compare) == 0:
- flagged[:] = False
- else:
- for f in flags_to_compare:
- if not self.isValidFlag(f):
- raise ValueError(f"invalid flag: {f}")
- flagged &= cp(flags, f)
-
- return flagged
-
- def copy(self, **kwargs) -> CompatFlagger:
- copy = self._constructor()
- copy._flags = self._flags.copy(deep=True)
- return copy
-
- def isValidFlag(self, flag: FlagT) -> bool:
- """
- Check if given flag is known to this flagger.
-
- Parameters
- ----------
- flag: str
- The flag to be checked.
-
- Returns
- -------
- bool
- """
- return isinstance(flag, (float, int))
-
- def replaceField(self, field: str, flags: Optional[pd.Series], inplace: bool = False, **kwargs) -> CompatFlagger:
- """ Replace or delete all data for a given field.
-
- Parameters
- ----------
- field : str
- The field to replace / delete. If the field already exist, the respected data
- is replaced, otherwise the data is inserted in the respected field column.
- flags : pandas.Series or None
- If None, the series denoted by `field` will be deleted. Otherwise
- a series of flags (dtype flagger.dtype) that will replace the series
- currently stored under `field`
- inplace : bool, default False
- If False, a flagger copy is returned, otherwise the flagger is not copied.
- **kwargs : dict
- ignored.
-
- Returns
- -------
- flagger: saqc.flagger.BaseFlagger
- The flagger object or a copy of it (if inplace=True).
-
- Raises
- ------
- ValueError: (delete) if field does not exist
- TypeError: (replace / insert) if flags are not pd.Series
- """
-
- assertScalar("field", field, optional=False)
-
- out = self if inplace else deepcopy(self)
-
- # delete
- if flags is None:
- if field not in self._flags.columns:
- raise ValueError(f"{field}: field does not exist")
- del out._flags[field]
-
- # insert / replace
- else:
- if not isinstance(flags, pd.Series):
- raise TypeError(f"`flags` must be pd.Series.")
- del out._flags[field]
- out._flags[field] = flags.astype(self.dtype)
- return out
-
- def _check_field(self, field: Union[str, Sequence[str]]) -> Union[str, Sequence[str]]:
- """ Check if (all) field(s) in self._flags. """
-
- # wait for outcome of
- # https://git.ufz.de/rdm-software/saqc/issues/46
- failed = []
- if isinstance(field, str):
- if field not in self._flags.columns:
- failed += [field]
- else:
- try:
- for f in field:
- if f not in self._flags.columns:
- failed += [f]
- # not iterable, probably a slice or
- # any indexer we dont have to check
- except TypeError:
- pass
-
- if failed:
- raise ValueError(f"key(s) missing in flags: {failed}")
- return field
-
- @property
- def UNFLAGGED(self) -> FlagT:
- """ Return the flag that indicates unflagged data """
- return UNFLAGGED
-
- @property
- def GOOD(self) -> FlagT:
- """ Return the flag that indicates the very best data """
- return UNFLAGGED
-
- @property
- def BAD(self) -> FlagT:
- """ Return the flag that indicates the worst data """
- return BAD
-
- def isSUSPICIOUS(self, flag: FlagT) -> bool:
- """ Return bool that indicates if the given flag is valid, but neither
- UNFLAGGED, BAD, nor GOOD."""
- return self.GOOD < flag < self.BAD
diff --git a/saqc/flagger/continuousflagger.py b/saqc/flagger/continuousflagger.py
deleted file mode 100644
index 37e96508d..000000000
--- a/saqc/flagger/continuousflagger.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import pandas as pd
-import numpy as np
-import intervals
-
-from saqc.flagger.baseflagger import BaseFlagger
-
-
-class ContinuousFlagger(BaseFlagger):
- def __init__(self, min_=0.0, max_=1.0, unflagged=-1.0):
- assert unflagged < 0 <= min_ < max_
- super().__init__(dtype=np.float64)
- self._interval = intervals.closed(min_, max_)
- self._unflagged_flag = unflagged
- self.signature = ("flag", "factor", "modify")
-
- def setFlags(self, field, loc=None, iloc=None, flag=None, force=False, factor=1, modify=False, **kwargs):
- # NOTE: incomplete, as the option to
- # update flags is temporarily gone
- return super().setFlags(field=field, loc=loc, iloc=iloc, flag=self._checkFlag(flag), force=force, **kwargs)
-
- # NOTE:
- # we should probably override _assureDtype here
-
- def _isDtype(self, flag):
- if isinstance(flag, pd.Series):
- # NOTE: it should be made sure, that all
- # values fall into the interval
- return flag.dtype == self.dtype
- return flag in self._interval or flag == self.UNFLAGGED
-
- @property
- def UNFLAGGED(self):
- return self._unflagged_flag
-
- @property
- def GOOD(self):
- return self._interval.lower
-
- @property
- def BAD(self):
- return self._interval.upper
-
- def isSUSPICIOUS(self, flag):
- return flag in intervals.open(self.GOOD, self.BAD)
diff --git a/saqc/flagger/dmpflagger.py b/saqc/flagger/dmpflagger.py
deleted file mode 100644
index a3f63a424..000000000
--- a/saqc/flagger/dmpflagger.py
+++ /dev/null
@@ -1,248 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-from __future__ import annotations
-
-import json
-from copy import deepcopy
-from typing import TypeVar, Optional, List
-
-import pandas as pd
-
-from dios import DictOfSeries
-
-from saqc.flagger.categoricalflagger import CategoricalFlagger
-from saqc.lib.tools import assertScalar, mergeDios
-
-DmpFlaggerT = TypeVar("DmpFlaggerT")
-
-
-class Keywords:
- VERSION = "$version"
-
-
-class FlagFields:
- FLAG = "quality_flag"
- CAUSE = "quality_cause"
- COMMENT = "quality_comment"
-
-
-class ColumnLevels:
- VARIABLES = "variables"
- FLAGS = "flags"
-
-
-FLAGS = ["NIL", "OK", "DOUBTFUL", "BAD"]
-
-
-class DmpFlagger(CategoricalFlagger):
- def __init__(self, **kwargs):
- super().__init__(FLAGS)
- self.flags_fields = [FlagFields.FLAG, FlagFields.CAUSE, FlagFields.COMMENT]
- self.extra_defaults = dict(cause=FLAGS[0], comment="")
- self.signature = self.signature + ("flag", "comment", "cause")
-
- self._global_comments = kwargs
- self._flags = None
- self._causes = None
- self._comments = None
-
- @property
- def causes(self):
- return self._causes
-
- @property
- def comments(self):
- return self._comments
-
- def toFrame(self):
- out = pd.concat(
- [self._flags.to_df(), self._causes.to_df(), self._comments.to_df()],
- axis=1,
- keys=[FlagFields.FLAG, FlagFields.CAUSE, FlagFields.COMMENT],
- )
- out = out.reorder_levels(order=[1, 0], axis=1).sort_index(axis=1, level=0, sort_remaining=False)
- return out
-
- def initFlags(self, data: DictOfSeries = None, flags: DictOfSeries = None):
- """
- initialize a flagger based on the given 'data' or 'flags'
- if 'data' is not None: return a flagger with flagger.UNFALGGED values
- if 'flags' is not None: return a flagger with the given flags
- """
-
- # implicit set self._flags, and make deepcopy of self aka. DmpFlagger
- newflagger = super().initFlags(data=data, flags=flags)
- newflagger._causes = newflagger._flags.astype(str)
- newflagger._comments = newflagger._flags.astype(str)
- newflagger._causes[:], newflagger._comments[:] = "", ""
- return newflagger
-
- def slice(self, field=None, loc=None, drop=None, inplace=False):
- newflagger = super().slice(field=field, loc=loc, drop=drop, inplace=inplace)
- flags = newflagger._flags
- newflagger._causes = self._causes.aloc[flags, ...]
- newflagger._comments = self._comments.aloc[flags, ...]
- return newflagger
-
- def merge(self, other: DmpFlagger, subset: Optional[List] = None, join: str = "merge", inplace=False):
- assert isinstance(other, DmpFlagger)
- flags = mergeDios(self._flags, other._flags, subset=subset, join=join)
- causes = mergeDios(self._causes, other._causes, subset=subset, join=join)
- comments = mergeDios(self._comments, other._comments, subset=subset, join=join)
- if inplace:
- self._flags = flags
- self._causes = causes
- self._comments = comments
- return self
- else:
- return self._construct_new(flags, causes, comments)
-
- def getFlags(self, field=None, loc=None, full=False):
- # loc should be a valid 2D-indexer and
- # then field must be None. Otherwise aloc
- # will fail and throw the correct Error.
- if isinstance(loc, DictOfSeries) and field is None:
- indexer = loc
- else:
- loc = slice(None) if loc is None else loc
- field = slice(None) if field is None else self._check_field(field)
- indexer = (loc, field)
-
- # this is a bug in `dios.aloc`, which may return a shallow copied dios, if `slice(None)` is passed
- # as row indexer. Thus is because pandas `.loc` return a shallow copy if a null-slice is passed to a series.
- flags = self._flags.aloc[indexer].copy()
-
- if full:
- causes = self._causes.aloc[indexer].copy()
- comments = self._comments.aloc[indexer].copy()
- return flags, dict(cause=causes, comment=comments)
- else:
- return flags
-
- def setFlags(
- self,
- field,
- loc=None,
- flag=None,
- cause="OTHER",
- comment="",
- force=False,
- inplace=False,
- with_extra=False,
- flag_after=None,
- flag_before=None,
- win_flag=None,
- **kwargs
- ):
- assert "iloc" not in kwargs, "deprecated keyword, iloc"
- assertScalar("field", self._check_field(field), optional=False)
-
- out = self if inplace else deepcopy(self)
-
- if with_extra:
- for val in [comment, cause, flag]:
- if not isinstance(val, pd.Series):
- raise TypeError(f"`flag`, `cause`, `comment` must be pd.Series, if `with_extra=True`.")
- assert flag.index.equals(comment.index) and flag.index.equals(cause.index)
-
- else:
- flag = self.BAD if flag is None else flag
- comment = json.dumps(
- {**self._global_comments,
- "comment": comment,
- "test": kwargs.get("func_name", "")}
- )
-
- flags = self.getFlags(field=field, loc=loc)
- if force:
- mask = pd.Series(True, index=flags.index, dtype=bool)
- else:
- mask = flags < flag
-
- # set flags of the test
- out._flags.aloc[mask, field] = flag
- out._causes.aloc[mask, field] = cause
- out._comments.aloc[mask, field] = comment
-
- # calc and set window flags
- if flag_after is not None or flag_before is not None:
- win_mask, win_flag = self._getWindowMask(field, mask, flag_after, flag_before, win_flag, flag, force)
- out._flags.aloc[win_mask, field] = win_flag
- out._causes.aloc[win_mask, field] = cause
- out._comments.aloc[win_mask, field] = comment
-
- return out
-
- def replaceField(self, field, flags, inplace=False, cause=None, comment=None, **kwargs):
- """ Replace or delete all data for a given field.
-
- Parameters
- ----------
- field : str
- The field to replace / delete. If the field already exist, the respected data
- is replaced, otherwise the data is inserted in the respected field column.
- flags : pandas.Series or None
- If None, the series denoted by `field` will be deleted. Otherwise
- a series of flags (dtype flagger.dtype) that will replace the series
- currently stored under `field`
- causes : pandas.Series
- A series of causes (dtype str).
- comments : pandas.Series
- A series of comments (dtype str).
- inplace : bool, default False
- If False, a flagger copy is returned, otherwise the flagger is not copied.
- **kwargs : dict
- ignored.
-
- Returns
- -------
- flagger: saqc.flagger.BaseFlagger
- The flagger object or a copy of it (if inplace=True).
-
- Raises
- ------
- ValueError: (delete) if field does not exist
- TypeError: (replace / insert) if flags, causes, comments are not pd.Series
- AssertionError: (replace / insert) if flags, causes, comments does not have the same index
-
- Notes
- -----
- If deletion is requested (`flags=None`), `causes` and `comments` are don't-care.
-
- Flags, causes and comments must have the same index, if flags is not None, also
- each is casted implicit to the respected dtype.
- """
- assertScalar("field", field, optional=False)
- out = self if inplace else deepcopy(self)
- causes, comments = cause, comment
-
- # delete
- if flags is None:
- if field not in self._flags:
- raise ValueError(f"{field}: field does not exist")
- del out._flags[field]
- del out._comments[field]
- del out._causes[field]
-
- # insert / replace
- else:
- for val in [flags, causes, comments]:
- if not isinstance(val, pd.Series):
- raise TypeError(f"`flag`, `cause`, `comment` must be pd.Series.")
- assert flags.index.equals(comments.index) and flags.index.equals(causes.index)
- out._flags[field] = flags.astype(self.dtype)
- out._causes[field] = causes.astype(str)
- out._comments[field] = comments.astype(str)
- return out
-
- def _construct_new(self, flags, causes, comments) -> DmpFlagger:
- new = DmpFlagger()
- new._global_comments = self._global_comments
- new._flags = flags
- new._causes = causes
- new._comments = comments
- return new
-
- @property
- def SUSPICIOUS(self):
- return FLAGS[-2]
diff --git a/saqc/flagger/positionalflagger.py b/saqc/flagger/positionalflagger.py
deleted file mode 100644
index 00af0b2b1..000000000
--- a/saqc/flagger/positionalflagger.py
+++ /dev/null
@@ -1,144 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import json
-from copy import deepcopy
-
-import pandas as pd
-
-from dios import DictOfSeries
-from saqc.flagger.baseflagger import BaseFlagger, COMPARATOR_MAP
-from saqc.flagger.dmpflagger import DmpFlagger
-from saqc.lib.tools import toSequence, assertScalar
-
-
-FLAGS = ("-1", "0", "1", "2")
-
-
-class PositionalFlagger(BaseFlagger):
- def __init__(self):
- super().__init__(dtype=str)
-
- def setFlags(
- self,
- field: str,
- loc=None,
- position=-1,
- flag=None,
- force: bool = False,
- inplace: bool = False,
- with_extra=False,
- flag_after=None,
- flag_before=None,
- win_flag=None,
- **kwargs
- ):
- assertScalar("field", field, optional=False)
-
- # prepping
- flag = str(self.BAD if flag is None else flag)
- self.isValidFlag(flag, fail=True)
- out = self if inplace else deepcopy(self)
- out_flags = out._flags[field]
-
- idx = self.getFlags(field, loc).index
- mask = pd.Series(True, index=idx, dtype=bool)
- mask = mask.reindex_like(out_flags).fillna(False)
-
- # replace unflagged with the magic starter '9'
- out_flags = out_flags.str.replace(f"^{self.UNFLAGGED}", "9", regex=True)
-
- # bring all flags to the desired length
- # length = position # if position > 0 else out_flags.str.len().max()
- if position == -1:
- length = position = out_flags.str.len().max()
- else:
- length = position = position + 1
- out_flags = out_flags.str.pad(length + 1, fillchar=self.GOOD, side="right")
-
- # we rigorously overwrite existing flags
- new_flags = out_flags.str[position]
- new_flags.loc[mask] = flag
-
- # calc window flags
- if flag_after is not None or flag_before is not None:
- win_mask, win_flag = self._getWindowMask(field, mask, flag_after, flag_before, win_flag, flag, force)
- new_flags.loc[win_mask] = win_flag
-
- out._flags[field] = out_flags.str[:position] + new_flags + out_flags.str[position+1:]
- return out
-
- def isFlagged(self, field=None, loc=None, flag=None, comparator=">"):
-
- field = slice(None) if field is None else field
- flags = self._getMaxFlag(field, loc).astype(int)
- flags = flags.loc[:, field]
-
- # notna() to prevent nans to become True,
- # eg.: `np.nan != 0 -> True`
- flagged = flags.notna()
- flags_to_compare = set(toSequence(flag, self.GOOD))
- if not flags_to_compare:
- flagged[:] = False
- return flagged
-
- cp = COMPARATOR_MAP[comparator]
- for f in flags_to_compare:
- self.isValidFlag(f, fail=True)
- flagged &= cp(flags, int(f))
- return flagged
-
- def isValidFlag(self, flag, fail=False):
- check = flag in FLAGS
- if check is False and fail is True:
- raise ValueError(f"invalid flag {flag}, given values should be in '{FLAGS}'")
- return check
-
- def _getMaxFlag(self, field, loc):
-
- data = {}
- flags = self.getFlags(field, loc)
- if isinstance(flags, pd.Series):
- flags = flags.to_frame()
- for col_name, col in flags.iteritems():
- mask = col != self.UNFLAGGED
- col = col.str.replace("^9", "0", regex=True)
- col[mask] = col[mask].apply(lambda x: max(list(x)))
- data[col_name] = col
- return DictOfSeries(data)
-
- def toDmpFlagger(self):
- self = PositionalFlagger().initFlags(flags=self._flags)
- dmp_flagger = DmpFlagger().initFlags(data=self._flags)
- flag_map = {
- self.BAD: dmp_flagger.BAD,
- self.SUSPICIOUS: dmp_flagger.SUSPICIOUS,
- self.GOOD: dmp_flagger.GOOD,
- }
- for pos_flag, dmp_flag in flag_map.items():
- loc = self.isFlagged(flag=pos_flag, comparator="==")
- dmp_flagger._flags.aloc[loc] = dmp_flag
-
- dmp_flagger._comments.loc[:] = self._flags.to_df().applymap(lambda v: json.dumps({"flag": v}))
- dmp_flagger._causes.loc[:] = "OTHER"
- return dmp_flagger
-
- @property
- def UNFLAGGED(self):
- return FLAGS[0]
-
- @property
- def GOOD(self):
- return FLAGS[1]
-
- @property
- def SUSPICIOUS(self):
- return FLAGS[2]
-
- @property
- def BAD(self):
- return FLAGS[3]
-
- def isSUSPICIOUS(self, flag):
- return flag == self.SUSPICIOUS
-
diff --git a/saqc/flagger/simpleflagger.py b/saqc/flagger/simpleflagger.py
deleted file mode 100644
index 4cda7b7ef..000000000
--- a/saqc/flagger/simpleflagger.py
+++ /dev/null
@@ -1,12 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from saqc.flagger.categoricalflagger import CategoricalFlagger
-
-
-FLAGS = [-1, 0, 1]
-
-
-class SimpleFlagger(CategoricalFlagger):
- def __init__(self):
- super().__init__(FLAGS)
diff --git a/saqc/funcs/breaks.py b/saqc/funcs/breaks.py
index 3af127ecb..7349f7a6f 100644
--- a/saqc/funcs/breaks.py
+++ b/saqc/funcs/breaks.py
@@ -10,11 +10,11 @@ from typing import Tuple
from saqc.lib.tools import groupConsecutives
from saqc.funcs.changepoints import assignChangePointCluster
from saqc.core.register import register
-from saqc.flagger.baseflagger import BaseFlagger
+from saqc.flagger import Flagger
@register(masking='field', module="breaks")
-def flagMissing(data: DictOfSeries, field: str, flagger: BaseFlagger, nodata: float=np.nan, **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+def flagMissing(data: DictOfSeries, field: str, flagger: Flagger, nodata: float=np.nan, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
The function flags all values indicating missing data.
@@ -24,7 +24,7 @@ def flagMissing(data: DictOfSeries, field: str, flagger: BaseFlagger, nodata: fl
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
nodata : any, default np.nan
A value that defines missing data.
@@ -33,7 +33,7 @@ def flagMissing(data: DictOfSeries, field: str, flagger: BaseFlagger, nodata: fl
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
"""
@@ -49,7 +49,7 @@ def flagMissing(data: DictOfSeries, field: str, flagger: BaseFlagger, nodata: fl
@register(masking='field', module="breaks")
-def flagIsolated(data: DictOfSeries, field: str, flagger: BaseFlagger, gap_window: str, group_window: str, **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+def flagIsolated(data: DictOfSeries, field: str, flagger: Flagger, gap_window: str, group_window: str, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
The function flags arbitrary large groups of values, if they are surrounded by sufficiently
large data gaps. A gap is defined as group of missing and/or flagged values.
@@ -67,7 +67,7 @@ def flagIsolated(data: DictOfSeries, field: str, flagger: BaseFlagger, gap_windo
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional informations related to `data`.
gap_window : str
The minimum size of the gap before and after a group of valid values, making this group considered an
@@ -80,7 +80,7 @@ def flagIsolated(data: DictOfSeries, field: str, flagger: BaseFlagger, gap_windo
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
"""
@@ -109,7 +109,7 @@ def flagIsolated(data: DictOfSeries, field: str, flagger: BaseFlagger, gap_windo
@register(masking='field', module="breaks")
-def flagJumps(data: DictOfSeries, field: str, flagger: BaseFlagger, thresh: float, winsz: str, min_periods: int=1,
+def flagJumps(data: DictOfSeries, field: str, flagger: Flagger, thresh: float, winsz: str, min_periods: int=1,
**kwargs):
"""
Flag datapoints, where the mean of the values significantly changes (where the value course "jumps").
diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py
index 191470ec0..0bca70f2d 100644
--- a/saqc/funcs/changepoints.py
+++ b/saqc/funcs/changepoints.py
@@ -13,13 +13,13 @@ from dios import DictOfSeries
from saqc.core.register import register
from saqc.lib.tools import customRoller
-from saqc.flagger.baseflagger import BaseFlagger
+from saqc.flagger import Flagger
logger = logging.getLogger("SaQC")
@register(masking='field', module="changepoints")
-def flagChangePoints(data: DictOfSeries, field: str, flagger: BaseFlagger,
+def flagChangePoints(data: DictOfSeries, field: str, flagger: Flagger,
stat_func: Callable[[np.array], np.array],
thresh_func: Callable[[np.array], np.array],
bwd_window: str,
@@ -30,7 +30,7 @@ def flagChangePoints(data: DictOfSeries, field: str, flagger: BaseFlagger,
try_to_jit: bool=True,
reduce_window: str=None,
reduce_func: Callable[[np.array, np.array], np.array]=lambda x, y: x.argmax(),
- **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+ **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
Flag datapoints, where the parametrization of the process, the data is assumed to generate by, significantly
changes.
@@ -92,7 +92,7 @@ def flagChangePoints(data: DictOfSeries, field: str, flagger: BaseFlagger,
@register(masking='field', module="changepoints")
-def assignChangePointCluster(data: DictOfSeries, field: str, flagger: BaseFlagger,
+def assignChangePointCluster(data: DictOfSeries, field: str, flagger: Flagger,
stat_func: Callable[[np.array, np.array], float],
thresh_func: Callable[[np.array, np.array], float],
bwd_window: str,
@@ -106,7 +106,7 @@ def assignChangePointCluster(data: DictOfSeries, field: str, flagger: BaseFlagge
model_by_resids: bool=False,
flag_changepoints: bool=False,
assign_cluster: bool=True,
- **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+ **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
Assigns label to the data, aiming to reflect continous regimes of the processes the data is assumed to be
diff --git a/saqc/funcs/constants.py b/saqc/funcs/constants.py
index 204c70703..9244d8c82 100644
--- a/saqc/funcs/constants.py
+++ b/saqc/funcs/constants.py
@@ -10,13 +10,13 @@ import pandas as pd
from dios import DictOfSeries
from saqc.core.register import register
-from saqc.flagger.baseflagger import BaseFlagger
+from saqc.flagger import Flagger
from saqc.lib.ts_operators import varQC
from saqc.lib.tools import customRoller, getFreqDelta
@register(masking='field', module="constants")
-def flagConstants(data: DictOfSeries, field: str, flagger: BaseFlagger, thresh: float, window: str, **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+def flagConstants(data: DictOfSeries, field: str, flagger: Flagger, thresh: float, window: str, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
This functions flags plateaus/series of constant values of length `window` if
their maximum total change is smaller than thresh.
@@ -34,7 +34,7 @@ def flagConstants(data: DictOfSeries, field: str, flagger: BaseFlagger, thresh:
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
thresh : float
Upper bound for the maximum total change of an interval to be flagged constant.
@@ -45,7 +45,7 @@ def flagConstants(data: DictOfSeries, field: str, flagger: BaseFlagger, thresh:
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional informations related to `data`.
Flags values may have changed, relatively to the flagger input.
"""
@@ -72,10 +72,10 @@ def flagConstants(data: DictOfSeries, field: str, flagger: BaseFlagger, thresh:
@register(masking='field', module="constants")
def flagByVariance(
- data: DictOfSeries, field: str, flagger: BaseFlagger,
+ data: DictOfSeries, field: str, flagger: Flagger,
window: str="12h", thresh: float=0.0005,
max_missing: int=None, max_consec_missing: int=None, **kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
Function flags plateaus/series of constant values. Any interval of values y(t),..y(t+n) is flagged, if:
@@ -89,7 +89,7 @@ def flagByVariance(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
window : str
Only intervals of minimum size "window" have the chance to get flagged as constant intervals
@@ -108,7 +108,7 @@ def flagByVariance(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional informations related to `data`.
Flags values may have changed, relatively to the flagger input.
"""
diff --git a/saqc/funcs/curvefit.py b/saqc/funcs/curvefit.py
index bb39d42fb..c4862fe87 100644
--- a/saqc/funcs/curvefit.py
+++ b/saqc/funcs/curvefit.py
@@ -14,20 +14,20 @@ from dios import DictOfSeries
from saqc.core.register import register
from saqc.lib.tools import getFreqDelta
-from saqc.flagger.baseflagger import BaseFlagger
+from saqc.flagger import Flagger
from saqc.lib.ts_operators import polyRollerIrregular, polyRollerNumba, polyRoller, polyRollerNoMissingNumba, polyRollerNoMissing
@register(masking='field', module="curvefit")
-def fitPolynomial(data: DictOfSeries, field: str, flagger: BaseFlagger,
+def fitPolynomial(data: DictOfSeries, field: str, flagger: Flagger,
winsz: Union[int, str],
polydeg: int,
numba: Literal[True, False, "auto"]="auto",
eval_flags: bool=True,
min_periods: int=0,
return_residues: bool=False,
- **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+ **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
Function fits a polynomial model to the data and returns the fitted data curve.
@@ -68,7 +68,7 @@ def fitPolynomial(data: DictOfSeries, field: str, flagger: BaseFlagger,
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-modelled.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
winsz : {str, int}
The size of the window you want to use for fitting. If an integer is passed, the size
@@ -98,7 +98,7 @@ def fitPolynomial(data: DictOfSeries, field: str, flagger: BaseFlagger,
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
diff --git a/saqc/funcs/drift.py b/saqc/funcs/drift.py
index 1c49dcc96..9c7c26b58 100644
--- a/saqc/funcs/drift.py
+++ b/saqc/funcs/drift.py
@@ -13,7 +13,7 @@ from scipy.spatial.distance import pdist
from dios import DictOfSeries
from saqc.core.register import register
-from saqc.flagger.baseflagger import BaseFlagger
+from saqc.flagger import Flagger
from saqc.funcs.resampling import shift
from saqc.funcs.changepoints import assignChangePointCluster
from saqc.funcs.tools import drop, copy
@@ -22,14 +22,14 @@ from saqc.lib.ts_operators import expModelFunc
@register(masking='all', module="drift")
-def flagDriftFromNorm(data: DictOfSeries, field: str, flagger: BaseFlagger,
+def flagDriftFromNorm(data: DictOfSeries, field: str, flagger: Flagger,
fields: Sequence[str],
segment_freq: str,
norm_spread: float,
norm_frac: float=0.5,
metric: Callable[[np.array, np.array], float]=lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
linkage_method: Literal["single", "complete", "average", "weighted", "centroid", "median", "ward"]="single",
- **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+ **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
The function flags value courses that significantly deviate from a group of normal value courses.
@@ -45,7 +45,7 @@ def flagDriftFromNorm(data: DictOfSeries, field: str, flagger: BaseFlagger,
A dictionary of pandas.Series, holding all the data.
field : str
A dummy parameter.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional informations related to `data`.
fields : str
List of fieldnames in data, determining which variables are to be included into the flagging process.
@@ -75,7 +75,7 @@ def flagDriftFromNorm(data: DictOfSeries, field: str, flagger: BaseFlagger,
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the input flagger.
@@ -131,12 +131,12 @@ def flagDriftFromNorm(data: DictOfSeries, field: str, flagger: BaseFlagger,
@register(masking='all', module="drift")
-def flagDriftFromReference(data: DictOfSeries, field: str, flagger: BaseFlagger,
+def flagDriftFromReference(data: DictOfSeries, field: str, flagger: Flagger,
fields: Sequence[str],
segment_freq: str,
thresh: float,
metric: Callable[[np.array, np.array], float]=lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
- **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+ **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
The function flags value courses that deviate from a reference course by a margin exceeding a certain threshold.
@@ -148,7 +148,7 @@ def flagDriftFromReference(data: DictOfSeries, field: str, flagger: BaseFlagger,
A dictionary of pandas.Series, holding all the data.
field : str
The reference variable, the deviation from wich determines the flagging.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional informations related to `data`.
fields : str
List of fieldnames in data, determining wich variables are to be included into the flagging process.
@@ -167,7 +167,7 @@ def flagDriftFromReference(data: DictOfSeries, field: str, flagger: BaseFlagger,
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the input flagger.
@@ -200,7 +200,7 @@ def flagDriftFromReference(data: DictOfSeries, field: str, flagger: BaseFlagger,
@register(masking='all', module="drift")
-def flagDriftFromScaledNorm(data: DictOfSeries, field: str, flagger: BaseFlagger,
+def flagDriftFromScaledNorm(data: DictOfSeries, field: str, flagger: Flagger,
fields_scale1: Sequence[str],
fields_scale2: Sequence[str],
segment_freq: str,
@@ -208,7 +208,7 @@ def flagDriftFromScaledNorm(data: DictOfSeries, field: str, flagger: BaseFlagger
norm_frac: float=0.5,
metric: Callable[[np.array, np.array], float]=lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
linkage_method: Literal["single", "complete", "average", "weighted", "centroid", "median", "ward"]="single",
- **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+ **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -312,9 +312,9 @@ def flagDriftFromScaledNorm(data: DictOfSeries, field: str, flagger: BaseFlagger
@register(masking='all', module="drift")
-def correctExponentialDrift(data: DictOfSeries, field: str, flagger: BaseFlagger,
+def correctExponentialDrift(data: DictOfSeries, field: str, flagger: Flagger,
maint_data_field: str, cal_mean: int=5, flag_maint_period: bool=False,
- **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+ **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
The function fits an exponential model to chunks of data[field].
It is assumed, that between maintenance events, there is a drift effect shifting the meassurements in a way, that
@@ -351,7 +351,7 @@ def correctExponentialDrift(data: DictOfSeries, field: str, flagger: BaseFlagger
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to correct.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
maint_data_field : str
The fieldname of the datacolumn holding the maintenance information.
@@ -369,7 +369,7 @@ def correctExponentialDrift(data: DictOfSeries, field: str, flagger: BaseFlagger
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
"""
@@ -412,11 +412,11 @@ def correctExponentialDrift(data: DictOfSeries, field: str, flagger: BaseFlagger
@register(masking='all', module="drift")
-def correctRegimeAnomaly(data: DictOfSeries, field: str, flagger: BaseFlagger,
+def correctRegimeAnomaly(data: DictOfSeries, field: str, flagger: Flagger,
cluster_field: str,
model: Callable[[np.array, Any], np.array],
regime_transmission: Optional[str]=None,
- x_date: bool=False) -> Tuple[DictOfSeries, BaseFlagger]:
+ x_date: bool=False) -> Tuple[DictOfSeries, Flagger]:
"""
Function fits the passed model to the different regimes in data[field] and tries to correct
those values, that have assigned a negative label by data[cluster_field].
@@ -521,14 +521,14 @@ def correctRegimeAnomaly(data: DictOfSeries, field: str, flagger: BaseFlagger,
@register(masking='all', module="drift")
-def correctOffset(data: DictOfSeries, field: str, flagger: BaseFlagger,
+def correctOffset(data: DictOfSeries, field: str, flagger: Flagger,
max_mean_jump: float,
normal_spread: float,
search_winsz: str,
min_periods: int,
regime_transmission: Optional[str]=None,
**kwargs
- ) -> Tuple[DictOfSeries, BaseFlagger]:
+ ) -> Tuple[DictOfSeries, Flagger]:
"""
Parameters
@@ -610,13 +610,13 @@ def _drift_fit(x, shift_target, cal_mean):
@register(masking='all', module="drift")
-def flagRegimeAnomaly(data: DictOfSeries, field: str, flagger: BaseFlagger,
+def flagRegimeAnomaly(data: DictOfSeries, field: str, flagger: Flagger,
cluster_field: str,
norm_spread: float,
linkage_method: Literal["single", "complete", "average", "weighted", "centroid", "median", "ward"]="single",
metric: Callable[[np.array, np.array], float]=lambda x, y: np.abs(np.nanmean(x) - np.nanmean(y)),
norm_frac: float=0.5,
- **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+ **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
A function to flag values belonging to an anomalous regime regarding modelling regimes of field.
@@ -671,7 +671,7 @@ def flagRegimeAnomaly(data: DictOfSeries, field: str, flagger: BaseFlagger,
@register(masking='all', module="drift")
-def assignRegimeAnomaly(data: DictOfSeries, field: str, flagger: BaseFlagger,
+def assignRegimeAnomaly(data: DictOfSeries, field: str, flagger: Flagger,
cluster_field: str,
norm_spread: float,
linkage_method: Literal["single", "complete", "average", "weighted", "centroid", "median", "ward"]="single",
@@ -679,7 +679,7 @@ def assignRegimeAnomaly(data: DictOfSeries, field: str, flagger: BaseFlagger,
norm_frac: float=0.5,
set_cluster: bool=True,
set_flags: bool=False,
- **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+ **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
A function to detect values belonging to an anomalous regime regarding modelling regimes of field.
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index b18714373..92495b887 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -8,23 +8,23 @@ import pandas as pd
from dios import DictOfSeries
from saqc.core.register import register
-from saqc.flagger.baseflagger import BaseFlagger
+from saqc.flagger import Flagger
@register(masking='field', module="flagtools")
-def clearFlags(data: DictOfSeries, field: str, flagger: BaseFlagger, **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+def clearFlags(data: DictOfSeries, field: str, flagger: Flagger, **kwargs) -> Tuple[DictOfSeries, Flagger]:
flagger = flagger.clearFlags(field, **kwargs)
return data, flagger
@register(masking='field', module="flagtools")
-def forceFlags(data: DictOfSeries, field: str, flagger: BaseFlagger, flag: Any, **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+def forceFlags(data: DictOfSeries, field: str, flagger: Flagger, flag: Any, **kwargs) -> Tuple[DictOfSeries, Flagger]:
flagger = flagger.clearFlags(field).setFlags(field, flag=flag, inplace=True, **kwargs)
return data, flagger
@register(masking='field', module="flagtools")
-def flagDummy(data: DictOfSeries, field: str, flagger: BaseFlagger, **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+def flagDummy(data: DictOfSeries, field: str, flagger: Flagger, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
Function does nothing but returning data and flagger.
@@ -34,21 +34,21 @@ def flagDummy(data: DictOfSeries, field: str, flagger: BaseFlagger, **kwargs) -
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional informations related to `data`.
Returns
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
"""
return data, flagger
@register(masking='field', module="flagtools")
-def flagForceFail(data: DictOfSeries, field: str, flagger: BaseFlagger, **kwargs):
+def flagForceFail(data: DictOfSeries, field: str, flagger: Flagger, **kwargs):
"""
Function raises a runtime error.
@@ -58,7 +58,7 @@ def flagForceFail(data: DictOfSeries, field: str, flagger: BaseFlagger, **kwargs
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional informations related to `data`.
"""
@@ -66,7 +66,7 @@ def flagForceFail(data: DictOfSeries, field: str, flagger: BaseFlagger, **kwargs
@register(masking='field', module="flagtools")
-def flagUnflagged(data: DictOfSeries, field: str, flagger: BaseFlagger, flag: Optional[Any]=None, **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+def flagUnflagged(data: DictOfSeries, field: str, flagger: Flagger, flag: Optional[Any]=None, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
Function sets the flagger.GOOD flag to all values flagged better then flagger.GOOD.
If there is an entry 'flag' in the kwargs dictionary passed, the
@@ -78,7 +78,7 @@ def flagUnflagged(data: DictOfSeries, field: str, flagger: BaseFlagger, flag: Op
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional informations related to `data`.
kwargs : Dict
If kwargs contains 'flag' entry, kwargs['flag] is set, if no entry 'flag' is present,
@@ -88,7 +88,7 @@ def flagUnflagged(data: DictOfSeries, field: str, flagger: BaseFlagger, flag: Op
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
"""
@@ -98,7 +98,7 @@ def flagUnflagged(data: DictOfSeries, field: str, flagger: BaseFlagger, flag: Op
@register(masking='field', module="flagtools")
-def flagGood(data: DictOfSeries, field: str, flagger: BaseFlagger, flag: Optional[Any]=None, **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+def flagGood(data: DictOfSeries, field: str, flagger: Flagger, flag: Optional[Any]=None, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
Function sets the flagger.GOOD flag to all values flagged better then flagger.GOOD.
@@ -108,14 +108,14 @@ def flagGood(data: DictOfSeries, field: str, flagger: BaseFlagger, flag: Optiona
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional informations related to `data`.
Returns
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
"""
@@ -124,12 +124,12 @@ def flagGood(data: DictOfSeries, field: str, flagger: BaseFlagger, flag: Optiona
@register(masking='field', module="flagtools")
def flagManual(
- data: DictOfSeries, field: str, flagger: BaseFlagger,
+ data: DictOfSeries, field: str, flagger: Flagger,
mdata: Union[pd.Series, pd.DataFrame, DictOfSeries],
mflag: Any = 1,
method=Literal["plain", "ontime", "left-open", "right-open"],
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
Flag data by given, "manually generated" data.
@@ -144,7 +144,7 @@ def flagManual(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional informations related to `data`.
mdata : {pd.Series, pd.Dataframe, DictOfSeries}
The "manually generated" data
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index 22bddb109..3fc4d8305 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -12,17 +12,17 @@ from dios import DictOfSeries
from saqc.core.register import register
from saqc.core.visitor import ENVIRONMENT
-from saqc.flagger.baseflagger import BaseFlagger
+from saqc.flagger import Flagger
-def _dslIsFlagged(flagger: BaseFlagger, var: pd.Series, flag: Any=None, comparator: str=">=") -> Union[pd.Series, DictOfSeries]:
+def _dslIsFlagged(flagger: Flagger, var: pd.Series, flag: Any=None, comparator: str=">=") -> Union[pd.Series, DictOfSeries]:
"""
helper function for `flag`
"""
return flagger.isFlagged(var.name, flag=flag, comparator=comparator)
-def _execGeneric(flagger: BaseFlagger, data: DictOfSeries, func: Callable[[pd.Series], pd.Series], field: str, nodata: float) -> pd.Series:
+def _execGeneric(flagger: Flagger, data: DictOfSeries, func: Callable[[pd.Series], pd.Series], field: str, nodata: float) -> pd.Series:
# TODO:
# - check series.index compatibility
# - field is only needed to translate 'this' parameters
@@ -52,7 +52,7 @@ def _execGeneric(flagger: BaseFlagger, data: DictOfSeries, func: Callable[[pd.Se
@register(masking='all', module="generic")
-def process(data: DictOfSeries, field: str, flagger: BaseFlagger, func: Callable[[pd.Series], pd.Series], nodata: float=np.nan, **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+def process(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd.Series], pd.Series], nodata: float=np.nan, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
generate/process data with generically defined functions.
@@ -73,7 +73,7 @@ def process(data: DictOfSeries, field: str, flagger: BaseFlagger, func: Callable
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, where you want the result from the generic expressions processing to be written to.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
func : Callable
The data processing function with parameter names that will be
@@ -87,7 +87,7 @@ def process(data: DictOfSeries, field: str, flagger: BaseFlagger, func: Callable
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
The shape of the data may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
The flags shape may have changed relatively to the input flagger.
@@ -121,7 +121,7 @@ def process(data: DictOfSeries, field: str, flagger: BaseFlagger, func: Callable
@register(masking='all', module="generic")
-def flag(data: DictOfSeries, field: str, flagger: BaseFlagger, func: Callable[[pd.Series], pd.Series], nodata: float=np.nan, **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+def flag(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd.Series], pd.Series], nodata: float=np.nan, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
a function to flag a data column by evaluation of a generic expression.
@@ -147,7 +147,7 @@ def flag(data: DictOfSeries, field: str, flagger: BaseFlagger, func: Callable[[p
field : str
The fieldname of the column, where you want the result from the generic expressions evaluation to be projected
to.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
func : Callable
The expression that is to be evaluated is passed in form of a callable, with parameter names that will be
@@ -160,7 +160,7 @@ def flag(data: DictOfSeries, field: str, flagger: BaseFlagger, func: Callable[[p
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index a8b9c7164..a45b272d5 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -10,7 +10,7 @@ import pandas as pd
from dios import DictOfSeries
from saqc.core.register import register
-from saqc.flagger.baseflagger import BaseFlagger
+from saqc.flagger import Flagger
from saqc.lib.tools import toSequence, evalFreqStr, dropper
from saqc.lib.ts_operators import interpolateNANs
@@ -18,14 +18,14 @@ from saqc.lib.ts_operators import interpolateNANs
@register(masking='field', module="interpolation")
def interpolateByRolling(
- data: DictOfSeries, field: str, flagger: BaseFlagger,
+ data: DictOfSeries, field: str, flagger: Flagger,
winsz: Union[str, int],
func: Callable[[pd.Series], float]=np.median,
center: bool=True,
min_periods: int=0,
interpol_flag=Any,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
Interpolates missing values (nan values present in the data) by assigning them the aggregation result of
a window surrounding them.
@@ -39,7 +39,7 @@ def interpolateByRolling(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-interpolated.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
winsz : int, str
The size of the window, the aggregation is computed from. Either counted in periods number (Integer passed),
@@ -60,7 +60,7 @@ def interpolateByRolling(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
"""
@@ -93,7 +93,7 @@ def interpolateByRolling(
def interpolateInvalid(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
method: Literal["linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric", "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"],
inter_order: int=2,
inter_limit: int=2,
@@ -101,7 +101,7 @@ def interpolateInvalid(
downgrade_interpolation: bool=False,
not_interpol_flags: Optional[Union[Any, Sequence[Any]]]=None,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
Function to interpolate nan values in the data.
@@ -121,7 +121,7 @@ def interpolateInvalid(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-interpolated.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
method : {"linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric",
"polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"}: string
@@ -145,7 +145,7 @@ def interpolateInvalid(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
"""
@@ -183,7 +183,7 @@ def interpolateInvalid(
def interpolateIndex(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
freq: str,
method: Literal["linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric", "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"],
inter_order: int=2,
@@ -194,7 +194,7 @@ def interpolateIndex(
inter_limit: int=2,
freq_check: Optional[Literal["check", "auto"]]=None,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
Function to interpolate the data at regular (equidistant) timestamps (or Grid points).
@@ -216,7 +216,7 @@ def interpolateIndex(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-interpolated.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
freq : str
An Offset String, interpreted as the frequency of
@@ -256,7 +256,7 @@ def interpolateIndex(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index f76090e8e..a68d5f69c 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -15,7 +15,7 @@ from outliers import smirnov_grubbs
from dios import DictOfSeries
from saqc.core.register import register
-from saqc.flagger.baseflagger import BaseFlagger
+from saqc.flagger import Flagger
from saqc.lib.tools import (
customRoller,
findIndex,
@@ -29,13 +29,13 @@ import saqc.lib.ts_operators as ts_ops
def flagByStray(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
partition_freq: Optional[Union[str, int]]=None,
partition_min: int=11,
iter_start: float=0.5,
alpha: float=0.05,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
Flag outliers in 1-dimensional (score) data with the STRAY Algorithm.
@@ -47,7 +47,7 @@ def flagByStray(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
partition_freq : {None, str, int}, default None
partition_freq : {np.inf, float, str}, default np.inf
@@ -123,7 +123,7 @@ def flagByStray(
def _evalStrayLabels(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
fields: Sequence[str],
reduction_range: Optional[str]=None,
reduction_drop_flagged: bool=False,
@@ -131,7 +131,7 @@ def _evalStrayLabels(
reduction_min_periods: int=1,
at_least_one: bool=True,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
The function "reduces" an observations flag to components of it, by applying MAD (See references)
test onto every components temporal surrounding.
@@ -142,7 +142,7 @@ def _evalStrayLabels(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the labels to be evaluated.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
fields : list[str]
A list of strings, holding the column names of the variables, the stray labels shall be
@@ -337,7 +337,7 @@ def _expFit(val_frame, scoring_method="kNNMaxGap", n_neighbors=10, iter_start=0.
def flagMVScores(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
fields: Sequence[str],
trafo: Callable[[pd.Series], pd.Series]=lambda x: x,
alpha: float=0.05,
@@ -352,7 +352,7 @@ def flagMVScores(
reduction_thresh: float=3.5,
reduction_min_periods: int=1,
**kwargs,
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
The algorithm implements a 3-step outlier detection procedure for simultaneously flagging of higher dimensional
data (dimensions > 3).
@@ -367,7 +367,7 @@ def flagMVScores(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged. (Here a dummy, for structural reasons)
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
fields : List[str]
List of fieldnames, corresponding to the variables that are to be included into the flagging process.
@@ -419,7 +419,7 @@ def flagMVScores(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed, relatively to the flagger input.
@@ -478,7 +478,7 @@ def flagMVScores(
def flagRaise(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
thresh: float,
raise_window: str,
intended_freq: str,
@@ -488,7 +488,7 @@ def flagRaise(
min_slope_weight: float=0.8,
numba_boost: bool=True,
**kwargs,
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
The function flags raises and drops in value courses, that exceed a certain threshold
within a certain timespan.
@@ -506,7 +506,7 @@ def flagRaise(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
thresh : float
The threshold, for the total rise (thresh > 0), or total drop (thresh < 0), value courses must
@@ -531,7 +531,7 @@ def flagRaise(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed, relatively to the flagger input.
@@ -641,7 +641,7 @@ def flagRaise(
@register(masking='field', module="outliers")
-def flagMAD(data: DictOfSeries, field: str, flagger: BaseFlagger, window: str, z: float=3.5, **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+def flagMAD(data: DictOfSeries, field: str, flagger: Flagger, window: str, z: float=3.5, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -657,7 +657,7 @@ def flagMAD(data: DictOfSeries, field: str, flagger: BaseFlagger, window: str, z
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged. (Here a dummy, for structural reasons)
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
window : str
Offset string. Denoting the windows size that the "Z-scored" values have to lie in.
@@ -668,7 +668,7 @@ def flagMAD(data: DictOfSeries, field: str, flagger: BaseFlagger, window: str, z
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed, relatively to the flagger input.
@@ -703,13 +703,13 @@ def flagMAD(data: DictOfSeries, field: str, flagger: BaseFlagger, window: str, z
def flagOffset(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
thresh: float,
tolerance: float,
window: Union[int, str],
numba_kickin: int=200000,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
A basic outlier test that is designed to work for harmonized and not harmonized data.
@@ -734,7 +734,7 @@ def flagOffset(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged. (Here a dummy, for structural reasons)
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
thresh : float
Minimum difference between to values, to consider the latter one as a spike. See condition (1)
@@ -753,7 +753,7 @@ def flagOffset(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed, relatively to the flagger input.
@@ -824,13 +824,13 @@ def flagOffset(
def flagByGrubbs(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
winsz: Union[str, int],
alpha: float=0.05,
min_periods: int=8,
check_lagged: bool=False,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
The function flags values that are regarded outliers due to the grubbs test.
@@ -851,7 +851,7 @@ def flagByGrubbs(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
winsz : {int, str}
The size of the window you want to use for outlier testing. If an integer is passed, the size
@@ -871,7 +871,7 @@ def flagByGrubbs(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
@@ -927,11 +927,11 @@ def flagByGrubbs(
def flagRange(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
min: float=-np.inf,
max: float=np.inf,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
Function flags values not covered by the closed interval [`min`, `max`].
@@ -941,7 +941,7 @@ def flagRange(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
min : float
Lower bound for valid data.
@@ -952,7 +952,7 @@ def flagRange(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
"""
@@ -968,12 +968,12 @@ def flagRange(
def flagCrossStatistic(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
fields: Sequence[str],
thresh: float,
cross_stat: Literal["modZscore", "Zscore"]="modZscore",
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
Function checks for outliers relatively to the "horizontal" input data axis.
@@ -994,7 +994,7 @@ def flagCrossStatistic(
A dictionary of pandas.Series, holding all the data.
field : str
A dummy parameter.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional informations related to `data`.
fields : str
List of fieldnames in data, determining wich variables are to be included into the flagging process.
@@ -1011,7 +1011,7 @@ def flagCrossStatistic(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the input flagger.
diff --git a/saqc/funcs/pattern.py b/saqc/funcs/pattern.py
index ee65f6727..52437b89b 100644
--- a/saqc/funcs/pattern.py
+++ b/saqc/funcs/pattern.py
@@ -15,7 +15,7 @@ from mlxtend.evaluate import permutation_test
from dios.dios import DictOfSeries
from saqc.core.register import register
-from saqc.flagger.baseflagger import BaseFlagger
+from saqc.flagger import Flagger
from saqc.lib.tools import customRoller
@@ -23,12 +23,12 @@ from saqc.lib.tools import customRoller
def flagPatternByDTW(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
ref_field: str,
widths: Sequence[int]=(1, 2, 4, 8),
waveform: str="mexh",
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
Pattern recognition via wavelets.
@@ -44,7 +44,7 @@ def flagPatternByDTW(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to correct.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
ref_field: str
The fieldname in `data' which holds the pattern.
@@ -60,7 +60,7 @@ def flagPatternByDTW(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
@@ -113,7 +113,7 @@ def flagPatternByWavelet(
max_distance: float=0.03,
normalize: bool=True,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
""" Pattern Recognition via Dynamic Time Warping.
The steps are:
@@ -128,7 +128,7 @@ def flagPatternByWavelet(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to correct.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
ref_field: str
The fieldname in `data` which holds the pattern.
@@ -145,7 +145,7 @@ def flagPatternByWavelet(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index f43d5d292..d3a275314 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -12,7 +12,7 @@ import pandas as pd
from dios import DictOfSeries
from saqc.core.register import register
-from saqc.flagger.baseflagger import BaseFlagger
+from saqc.flagger import Flagger
from saqc.funcs.tools import copy, drop, rename
from saqc.funcs.interpolation import interpolateIndex
from saqc.lib.tools import dropper, evalFreqStr
@@ -36,14 +36,14 @@ METHOD2ARGS = {
def aggregate(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
freq: str,
value_func,
flag_func: Callable[[pd.Series], float]=np.nanmax,
method: Literal["fagg", "bagg", "nagg"]="nagg",
to_drop: Optional[Union[Any, Sequence[Any]]]=None,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
A method to "regularize" data by aggregating (resampling) data at a regular timestamp.
@@ -74,7 +74,7 @@ def aggregate(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-regularized.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.freq
freq : str
The sampling frequency the data is to be aggregated (resampled) at.
@@ -96,7 +96,7 @@ def aggregate(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
@@ -122,11 +122,11 @@ def aggregate(
def linear(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
freq: str,
to_drop: Optional[Union[Any, Sequence[Any]]]=None,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
A method to "regularize" data by interpolating linearly the data at regular timestamp.
@@ -148,7 +148,7 @@ def linear(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-regularized.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.freq
freq : str
An offset string. The frequency of the grid you want to interpolate your data at.
@@ -162,7 +162,7 @@ def linear(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
@@ -178,13 +178,13 @@ def linear(
def interpolate(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
freq: str,
method: Literal["linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric", "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"],
order: int=1,
to_drop: Optional[Union[Any, Sequence[Any]]]=None,
**kwargs,
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
A method to "regularize" data by interpolating the data at regular timestamp.
@@ -212,7 +212,7 @@ def interpolate(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-regularized.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.freq
freq : str
An offset string. The frequency of the grid you want to interpolate your data at.
@@ -232,7 +232,7 @@ def interpolate(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
@@ -256,11 +256,11 @@ def interpolate(
def mapToOriginal(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
method: Literal["inverse_fagg", "inverse_bagg", "inverse_nagg", "inverse_fshift", "inverse_bshift", "inverse_nshift", "inverse_interpolation"],
to_drop: Optional[Union[Any, Sequence[Any]]]=None,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
The Function function "undoes" regularization, by regaining the original data and projecting the
flags calculated for the regularized data onto the original ones.
@@ -305,7 +305,7 @@ def mapToOriginal(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-deharmonized.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.freq
method : {'inverse_fagg', 'inverse_bagg', 'inverse_nagg', 'inverse_fshift', 'inverse_bshift', 'inverse_nshift',
'inverse_interpolation'}
@@ -321,7 +321,7 @@ def mapToOriginal(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
@@ -337,14 +337,14 @@ def mapToOriginal(
def shift(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
freq: str,
method: Literal["fshift", "bshift", "nshift"]="nshift",
to_drop: Optional[Union[Any, Sequence[Any]]]=None,
empty_intervals_flag: Optional[str]=None,
freq_check: Optional[Literal["check", "auto"]]=None,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
data, flagger = copy(data, field, flagger, field + '_original')
data, flagger = _shift(
@@ -357,14 +357,14 @@ def shift(
def _shift(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
freq: str,
method: Literal["fshift", "bshift", "nshift"]="nshift",
to_drop: Optional[Union[Any, Sequence[Any]]]=None,
empty_intervals_flag: Optional[str]=None,
freq_check: Optional[Literal["check", "auto"]]=None,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
Function to shift data points to regular (equidistant) timestamps.
Values get shifted according to the keyword passed to the `method` parameter.
@@ -384,7 +384,7 @@ def _shift(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-shifted.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
freq : str
An frequency Offset String that will be interpreted as the sampling rate you want the data to be shifted to.
@@ -410,7 +410,7 @@ def _shift(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
@@ -446,7 +446,7 @@ def _shift(
def resample(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
freq: str,
agg_func: Callable[[pd.Series], pd.Series]=np.mean,
method: Literal["fagg", "bagg", "nagg"]="bagg",
@@ -460,7 +460,7 @@ def resample(
all_na_2_empty: bool=False,
freq_check: Optional[Literal["check", "auto"]]=None,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
Function to resample the data. Afterwards the data will be sampled at regular (equidistant) timestamps
(or Grid points). Sampling intervals therefor get aggregated with a function, specifyed by 'agg_func' parameter and
@@ -488,7 +488,7 @@ def resample(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-resampled.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
freq : str
An Offset String, that will be interpreted as the frequency you want to resample your data with.
@@ -538,7 +538,7 @@ def resample(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
@@ -595,14 +595,14 @@ def resample(
def reindexFlags(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
method: Literal["inverse_fagg", "inverse_bagg", "inverse_nagg", "inverse_fshift", "inverse_bshift", "inverse_nshift"],
source: str,
freq: Optional[str]=None,
to_drop: Optional[Union[Any, Sequence[Any]]]=None,
freq_check: Optional[Literal["check", "auto"]]=None,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
The Function projects flags of "source" onto flags of "field". Wherever the "field" flags are "better" then the
@@ -642,7 +642,7 @@ def reindexFlags(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to project the source-flags onto.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
method : {'inverse_fagg', 'inverse_bagg', 'inverse_nagg', 'inverse_fshift', 'inverse_bshift', 'inverse_nshift'}
The method used for projection of source flags onto field flags. See description above for more details.
@@ -664,7 +664,7 @@ def reindexFlags(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
diff --git a/saqc/funcs/residues.py b/saqc/funcs/residues.py
index 965575cf5..79a7a826c 100644
--- a/saqc/funcs/residues.py
+++ b/saqc/funcs/residues.py
@@ -9,7 +9,7 @@ import numpy as np
from dios import DictOfSeries
from saqc.core.register import register
-from saqc.flagger.baseflagger import BaseFlagger
+from saqc.flagger import Flagger
from saqc.funcs.rolling import roll
from saqc.funcs.curvefit import fitPolynomial
@@ -18,14 +18,14 @@ from saqc.funcs.curvefit import fitPolynomial
def calculatePolynomialResidues(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
winsz: Union[str, int],
polydeg: int,
numba: Literal[True, False, "auto"]="auto",
eval_flags: bool=True,
min_periods: Optional[int]=0,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
Function fits a polynomial model to the data and returns the residues.
@@ -66,7 +66,7 @@ def calculatePolynomialResidues(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-modelled.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
winsz : {str, int}
The size of the window you want to use for fitting. If an integer is passed, the size
@@ -94,7 +94,7 @@ def calculatePolynomialResidues(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
@@ -109,14 +109,14 @@ def calculatePolynomialResidues(
def calculateRollingResidues(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
winsz: Union[str, int],
func: Callable[[np.array], np.array]=np.mean,
eval_flags: bool=True,
min_periods: Optional[int]=0,
center: bool=True,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
data, flagger = roll(data, field, flagger, winsz, func=func, eval_flags=eval_flags,
min_periods=min_periods, center=center, return_residues=True, **kwargs)
diff --git a/saqc/funcs/rolling.py b/saqc/funcs/rolling.py
index fe8c0aa62..ab415bfe0 100644
--- a/saqc/funcs/rolling.py
+++ b/saqc/funcs/rolling.py
@@ -9,7 +9,7 @@ import pandas as pd
from dios import DictOfSeries
from saqc.core.register import register
-from saqc.flagger.baseflagger import BaseFlagger
+from saqc.flagger import Flagger
from saqc.lib.tools import getFreqDelta
@@ -17,7 +17,7 @@ from saqc.lib.tools import getFreqDelta
def roll(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
winsz: Union[str, int],
func: Callable[[pd.Series], float]=np.mean,
eval_flags: bool=True,
@@ -38,7 +38,7 @@ def roll(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-modelled.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
winsz : {int, str}
The size of the window you want to roll with. If an integer is passed, the size
@@ -65,7 +65,7 @@ def roll(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
"""
diff --git a/saqc/funcs/scores.py b/saqc/funcs/scores.py
index 3d82942ed..e723a1a91 100644
--- a/saqc/funcs/scores.py
+++ b/saqc/funcs/scores.py
@@ -9,7 +9,7 @@ import pandas as pd
from dios import DictOfSeries
from saqc.core.register import register
-from saqc.flagger.baseflagger import BaseFlagger
+from saqc.flagger import Flagger
from saqc.lib import ts_operators as ts_ops
from saqc.lib.tools import toSequence
@@ -18,7 +18,7 @@ from saqc.lib.tools import toSequence
def assignKNNScore(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
fields: Sequence[str],
n_neighbors: int=10,
trafo: Callable[[pd.Series], pd.Series]=lambda x: x,
@@ -32,7 +32,7 @@ def assignKNNScore(
p: int=2,
radius: Optional[float]=None,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
Score datapoints by an aggregation of the dictances to their k nearest neighbors.
diff --git a/saqc/funcs/tools.py b/saqc/funcs/tools.py
index fc1b55d41..4a900390c 100644
--- a/saqc/funcs/tools.py
+++ b/saqc/funcs/tools.py
@@ -9,12 +9,12 @@ import numpy as np
from dios import DictOfSeries
from saqc.core.register import register
-from saqc.flagger.baseflagger import BaseFlagger
+from saqc.flagger import Flagger
from saqc.lib.tools import periodicMask
@register(masking='none', module="tools")
-def copy(data: DictOfSeries, field: str, flagger: BaseFlagger, new_field: str, **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+def copy(data: DictOfSeries, field: str, flagger: Flagger, new_field: str, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
The function generates a copy of the data "field" and inserts it under the name field + suffix into the existing
data.
@@ -25,7 +25,7 @@ def copy(data: DictOfSeries, field: str, flagger: BaseFlagger, new_field: str, *
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to fork (copy).
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
new_field: str
Target name.
@@ -35,7 +35,7 @@ def copy(data: DictOfSeries, field: str, flagger: BaseFlagger, new_field: str, *
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
data shape may have changed relatively to the flagger input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags shape may have changed relatively to the flagger input.
"""
@@ -51,7 +51,7 @@ def copy(data: DictOfSeries, field: str, flagger: BaseFlagger, new_field: str, *
@register(masking='none', module="tools")
-def drop(data: DictOfSeries, field: str, flagger: BaseFlagger, **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+def drop(data: DictOfSeries, field: str, flagger: Flagger, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
The function drops field from the data dios and the flagger.
@@ -61,7 +61,7 @@ def drop(data: DictOfSeries, field: str, flagger: BaseFlagger, **kwargs) -> Tupl
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to drop.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
Returns
@@ -69,7 +69,7 @@ def drop(data: DictOfSeries, field: str, flagger: BaseFlagger, **kwargs) -> Tupl
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
data shape may have changed relatively to the flagger input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags shape may have changed relatively to the flagger input.
"""
@@ -81,7 +81,7 @@ def drop(data: DictOfSeries, field: str, flagger: BaseFlagger, **kwargs) -> Tupl
@register(masking='none', module="tools")
-def rename(data: DictOfSeries, field: str, flagger: BaseFlagger, new_name: str, **kwargs) -> Tuple[DictOfSeries, BaseFlagger]:
+def rename(data: DictOfSeries, field: str, flagger: Flagger, new_name: str, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
The function renames field to new name (in both, the flagger and the data).
@@ -91,7 +91,7 @@ def rename(data: DictOfSeries, field: str, flagger: BaseFlagger, new_name: str,
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to rename.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
new_name : str
String, field is to be replaced with.
@@ -100,7 +100,7 @@ def rename(data: DictOfSeries, field: str, flagger: BaseFlagger, new_name: str,
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
"""
# store
@@ -123,13 +123,13 @@ def rename(data: DictOfSeries, field: str, flagger: BaseFlagger, new_name: str,
def mask(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
mode: Literal["periodic", "mask_var"],
mask_var: Optional[str]=None,
period_start: Optional[str]=None,
period_end: Optional[str]=None,
include_bounds: bool=True
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
This function realizes masking within saqc.
@@ -155,7 +155,7 @@ def mask(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-masked.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
mode : {"periodic", "mask_var"}
The masking mode.
@@ -184,7 +184,7 @@ def mask(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
diff --git a/saqc/funcs/transformation.py b/saqc/funcs/transformation.py
index 61e368538..fdc99abbf 100644
--- a/saqc/funcs/transformation.py
+++ b/saqc/funcs/transformation.py
@@ -9,18 +9,18 @@ import pandas as pd
from dios import DictOfSeries
from saqc.core.register import register
-from saqc.flagger.baseflagger import BaseFlagger
+from saqc.flagger import Flagger
@register(masking='field', module="transformation")
def transform(
data: DictOfSeries,
field: str,
- flagger: BaseFlagger,
+ flagger: Flagger,
func: Callable[[pd.Series], pd.Series],
partition_freq: Optional[Union[float, str]]=None,
**kwargs
-) -> Tuple[DictOfSeries, BaseFlagger]:
+) -> Tuple[DictOfSeries, Flagger]:
"""
Function to transform data columns with a transformation that maps series onto series of the same length.
@@ -33,7 +33,7 @@ def transform(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-transformed.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
func : Callable[{pd.Series, np.array}, np.array]
Function to transform data[field] with.
@@ -50,7 +50,7 @@ def transform(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.BaseFlagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
"""
diff --git a/saqc/lib/plotting.py b/saqc/lib/plotting.py
index 0a9ac3066..1d3883fc5 100644
--- a/saqc/lib/plotting.py
+++ b/saqc/lib/plotting.py
@@ -8,7 +8,7 @@ import pandas as pd
import dios
import matplotlib.pyplot as plt
from typing import List, Dict, Optional
-from saqc.flagger import BaseFlagger
+from saqc.flagger import Flagger
def __importHelper():
@@ -88,8 +88,8 @@ def plotAllHook(
def plotHook(
data_old: Optional[dios.DictOfSeries],
data_new: dios.DictOfSeries,
- flagger_old: Optional[BaseFlagger],
- flagger_new: BaseFlagger,
+ flagger_old: Optional[Flagger],
+ flagger_new: Flagger,
sources: List[str],
targets: List[str],
plot_name: str = "",
@@ -119,8 +119,8 @@ def plotHook(
def _plotMultipleVariables(
data_old: Optional[dios.DictOfSeries],
data_new: dios.DictOfSeries,
- flagger_old: Optional[BaseFlagger],
- flagger_new: BaseFlagger,
+ flagger_old: Optional[Flagger],
+ flagger_new: Flagger,
targets: List[str],
show_info_table: bool = True,
annotations=None,
@@ -207,7 +207,7 @@ def _plotMultipleVariables(
def simplePlot(
data: dios.DictOfSeries,
- flagger: BaseFlagger,
+ flagger: Flagger,
field: str,
plot_name=None,
show_info_table: bool = True,
@@ -232,8 +232,8 @@ def simplePlot(
def _plotSingleVariable(
data_old: dios.DictOfSeries,
data_new: dios.DictOfSeries,
- flagger_old: BaseFlagger,
- flagger_new: BaseFlagger,
+ flagger_old: Flagger,
+ flagger_new: Flagger,
sources: List[str],
targets: List[str],
show_reference_data=True,
@@ -354,8 +354,8 @@ def _plotSingleVariable(
def _getDataFromVar(
data_old: dios.DictOfSeries,
data_new: dios.DictOfSeries,
- flagger_old: BaseFlagger,
- flagger_new: BaseFlagger,
+ flagger_old: Flagger,
+ flagger_new: Flagger,
varname: str,
):
"""
--
GitLab
From e263e20d2a6d33d3703dd5b494b686d6bdbe771f Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 26 Feb 2021 16:29:17 +0100
Subject: [PATCH 003/180] refactored flagger.BAD/GOOD/UNFLAGGED ->
BAD/GOOD/UNFLAGGED
---
saqc/common.py | 1 +
saqc/core/core.py | 16 ++++++++-------
saqc/core/visitor.py | 7 ++++---
saqc/flagger/__init__.py | 2 +-
saqc/funcs/changepoints.py | 4 ++--
saqc/funcs/flagtools.py | 9 ++++----
saqc/funcs/generic.py | 7 ++++---
saqc/funcs/interpolation.py | 7 ++++---
saqc/funcs/resampling.py | 41 +++++++++++++++++++------------------
saqc/funcs/scores.py | 2 +-
saqc/funcs/tools.py | 2 +-
saqc/lib/plotting.py | 13 ++++++------
saqc/lib/ts_operators.py | 2 +-
13 files changed, 61 insertions(+), 52 deletions(-)
diff --git a/saqc/common.py b/saqc/common.py
index 71d4cbb06..ce342c917 100644
--- a/saqc/common.py
+++ b/saqc/common.py
@@ -4,6 +4,7 @@ from numpy import nan as _nan
UNTOUCHED = _nan
UNFLAGGED = 0
+GOOD = 0
DOUBTFUL = 25
BAD = 255
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 0dae725a3..c843617da 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -20,7 +20,7 @@ import timeit
import inspect
from saqc.common import *
-from saqc.flagger.flags import init_flags_like, Flagger
+from saqc.flagger import init_flags_like, Flagger
from saqc.core.lib import APIController, ColumnSelector
from saqc.core.register import FUNC_MAP, SaQCFunction
from saqc.core.modules import FuncModules
@@ -164,7 +164,7 @@ class SaQC(FuncModules):
data, flagger = self._data, self._flagger
for selector, control, function in self._to_call:
- for sel, func in self._expandFields(selector, function, data.columns.union(flagger._flags.columns)):
+ for sel, func in self._expandFields(selector, function, data.columns.union(flagger.columns)):
logger.debug(f"processing: {sel.field}, {func.name}, {func.keywords}")
t0 = timeit.default_timer()
@@ -213,9 +213,11 @@ class SaQC(FuncModules):
realization = self.evaluate()
data, flagger = realization._data, realization._flagger
- if raw is False:
- return data.to_df(), flagger.toFrame()
- return data, flagger
+
+ if raw:
+ return data, flagger
+
+ return data.to_df(), flagger.toFrame()
def _wrap(self, func: SaQCFunction):
def inner(field: str, *fargs, target: str=None, regex: bool=False, to_mask=None, plot: bool=False, inplace: bool=False, **fkwargs) -> SaQC:
@@ -262,7 +264,7 @@ def _saqcCallFunc(locator, controller, function, data, flagger):
# NOTE:
# We assure that all columns in data have an equivalent column in flags,
# we might have more flagger columns though
- assert data.columns.difference(flagger.getFlags().columns).empty
+ assert data.columns.difference(flagger.columns).empty
field = locator.field
target = locator.target
@@ -286,7 +288,7 @@ def _saqcCallFunc(locator, controller, function, data, flagger):
# decorated by `register(masking='none')`, and so `to_mask` is ignored.
if masking == 'none' and to_mask not in (None, []):
logging.warning("`to_mask` is given, but the test ignore masking. Please refer to the documentation: TODO")
- to_mask = flagger.BAD if to_mask is None else to_mask
+ to_mask = BAD if to_mask is None else to_mask
data_in, mask = _maskData(data, flagger, columns, to_mask)
data_result, flagger_result = function(data_in, field, flagger)
diff --git a/saqc/core/visitor.py b/saqc/core/visitor.py
index c307a62ef..b9466c34c 100644
--- a/saqc/core/visitor.py
+++ b/saqc/core/visitor.py
@@ -6,6 +6,7 @@ import ast
import numpy as np
import pandas as pd
+from saqc.common import *
from saqc.core.register import FUNC_MAP
import saqc.lib.ts_operators as ts_ops
@@ -140,9 +141,9 @@ class ConfigFunctionParser(ast.NodeVisitor):
self.kwargs = {}
self.environment = {
- "GOOD": flagger.GOOD,
- "BAD": flagger.BAD,
- "UNFLAGGED": flagger.UNFLAGGED,
+ "GOOD": GOOD,
+ "BAD": BAD,
+ "UNFLAGGED": UNFLAGGED,
**ENVIRONMENT,
}
diff --git a/saqc/flagger/__init__.py b/saqc/flagger/__init__.py
index 890db7ee1..fd3f6f9f1 100644
--- a/saqc/flagger/__init__.py
+++ b/saqc/flagger/__init__.py
@@ -1,5 +1,5 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from .flags import Flagger
+from .flags import Flagger, init_flags_like
from .history import History
diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py
index 0bca70f2d..dcd41a66f 100644
--- a/saqc/funcs/changepoints.py
+++ b/saqc/funcs/changepoints.py
@@ -213,7 +213,7 @@ def assignChangePointCluster(data: DictOfSeries, field: str, flagger: Flagger,
residues = pd.Series(np.nan, index=data[field].index)
residues[masked_index] = stat_arr
data[field] = residues
- flagger = flagger.setFlags(field, flag=flagger.UNFLAGGED, force=True, **kwargs)
+ flagger = flagger.setFlags(field, flag=UNFLAGGED, force=True, **kwargs)
return data, flagger
det_index = masked_index[result_arr]
@@ -233,7 +233,7 @@ def assignChangePointCluster(data: DictOfSeries, field: str, flagger: Flagger,
# (better to start cluster labels with number one)
cluster += 1
data[field] = cluster
- flagger = flagger.setFlags(field, flag=flagger.UNFLAGGED, force=True, **kwargs)
+ flagger = flagger.setFlags(field, flag=UNFLAGGED, force=True, **kwargs)
if flag_changepoints:
flagger = flagger.setFlags(field, loc=det_index)
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index 92495b887..d50984cde 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -7,6 +7,7 @@ import pandas as pd
from dios import DictOfSeries
+from saqc.common import *
from saqc.core.register import register
from saqc.flagger import Flagger
@@ -68,7 +69,7 @@ def flagForceFail(data: DictOfSeries, field: str, flagger: Flagger, **kwargs):
@register(masking='field', module="flagtools")
def flagUnflagged(data: DictOfSeries, field: str, flagger: Flagger, flag: Optional[Any]=None, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
- Function sets the flagger.GOOD flag to all values flagged better then flagger.GOOD.
+ Function sets the GOOD flag to all values flagged better then GOOD.
If there is an entry 'flag' in the kwargs dictionary passed, the
function sets the kwargs['flag'] flag to all values flagged better kwargs['flag']
@@ -82,7 +83,7 @@ def flagUnflagged(data: DictOfSeries, field: str, flagger: Flagger, flag: Option
A flagger object, holding flags and additional informations related to `data`.
kwargs : Dict
If kwargs contains 'flag' entry, kwargs['flag] is set, if no entry 'flag' is present,
- 'flagger.UNFLAGGED' is set.
+ 'UNFLAGGED' is set.
Returns
-------
@@ -92,7 +93,7 @@ def flagUnflagged(data: DictOfSeries, field: str, flagger: Flagger, flag: Option
The flagger object, holding flags and additional Informations related to `data`.
"""
- flag = flagger.GOOD if flag is None else flag
+ flag = GOOD if flag is None else flag
flagger = flagger.setFlags(field, flag=flag, **kwargs)
return data, flagger
@@ -100,7 +101,7 @@ def flagUnflagged(data: DictOfSeries, field: str, flagger: Flagger, flag: Option
@register(masking='field', module="flagtools")
def flagGood(data: DictOfSeries, field: str, flagger: Flagger, flag: Optional[Any]=None, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
- Function sets the flagger.GOOD flag to all values flagged better then flagger.GOOD.
+ Function sets the GOOD flag to all values flagged better then GOOD.
Parameters
----------
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index 3fc4d8305..6bb3fdeb3 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -10,6 +10,7 @@ import pandas as pd
from dios import DictOfSeries
+from saqc.common import *
from saqc.core.register import register
from saqc.core.visitor import ENVIRONMENT
from saqc.flagger import Flagger
@@ -42,9 +43,9 @@ def _execGeneric(flagger: Flagger, data: DictOfSeries, func: Callable[[pd.Series
"mask": lambda cond: data[cond.name].mask(cond),
"this": field,
"NODATA": nodata,
- "GOOD": flagger.GOOD,
- "BAD": flagger.BAD,
- "UNFLAGGED": flagger.UNFLAGGED,
+ "GOOD": GOOD,
+ "BAD": BAD,
+ "UNFLAGGED": UNFLAGGED,
**ENVIRONMENT,
}
func.__globals__.update(globs)
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index a45b272d5..9453a5888 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -9,6 +9,7 @@ import pandas as pd
from dios import DictOfSeries
+from saqc.common import *
from saqc.core.register import register
from saqc.flagger import Flagger
@@ -237,7 +238,7 @@ def interpolateIndex(
empty_intervals_flag : str, default None
A Flag, that you want to assign to those values in the resulting equidistant sample grid, that were not
surrounded by valid data in the original dataset, and thus were not interpolated. Default automatically assigns
- ``flagger.BAD`` flag to those values.
+ ``BAD`` flag to those values.
grid_field : String, default None
Use the timestamp of another variable as (not necessarily regular) "grid" to be interpolated.
inter_limit : Integer, default 2
@@ -266,9 +267,9 @@ def interpolateIndex(
flagscol = flagger.getFlags(field)
freq = evalFreqStr(freq, freq_check, datcol.index)
if empty_intervals_flag is None:
- empty_intervals_flag = flagger.BAD
+ empty_intervals_flag = BAD
- drop_mask = dropper(field, to_drop, flagger, flagger.BAD)
+ drop_mask = dropper(field, to_drop, flagger, BAD)
drop_mask |= flagscol.isna()
drop_mask |= datcol.isna()
datcol[drop_mask] = np.nan
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index d3a275314..bf809340e 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -11,6 +11,7 @@ import pandas as pd
from dios import DictOfSeries
+from saqc.common import *
from saqc.core.register import register
from saqc.flagger import Flagger
from saqc.funcs.tools import copy, drop, rename
@@ -63,7 +64,7 @@ def aggregate(
Note, that, if there is no valid data (exisitng and not-na) available in a sampling interval assigned to a regular
timestamp by the selected method, nan gets assigned to this timestamp. The associated flag will be of value
- ``flagger.UNFLAGGED``.
+ ``UNFLAGGED``.
Note: the method will likely and significantly alter values and shape of ``data[field]``. The original data is kept
in the data dios and assigned to the fieldname ``field + '_original'``.
@@ -88,7 +89,7 @@ def aggregate(
"surrounding" interval). See description above for more details.
to_drop : {List[str], str}, default None
Flagtypes you want to drop before aggregation - effectively excluding values that are flagged
- with a flag in to_drop from the aggregation process. Default results in flagger.BAD
+ with a flag in to_drop from the aggregation process. Default results in BAD
values being dropped initially.
Returns
@@ -110,7 +111,7 @@ def aggregate(
agg_func=value_func,
flag_agg_func=flag_func,
method=method,
- empty_intervals_flag=flagger.UNFLAGGED,
+ empty_intervals_flag=UNFLAGGED,
to_drop=to_drop,
all_na_2_empty=True,
**kwargs,
@@ -140,7 +141,7 @@ def linear(
Note, that the data only gets interpolated at those (regular) timestamps, that have a valid (existing and
not-na) datapoint preceeding them and one succeeding them within freq range.
Regular timestamp that do not suffice this condition get nan assigned AND The associated flag will be of value
- ``flagger.UNFLAGGED``.
+ ``UNFLAGGED``.
Parameters
----------
@@ -154,7 +155,7 @@ def linear(
An offset string. The frequency of the grid you want to interpolate your data at.
to_drop : {List[str], str}, default None
Flagtypes you want to drop before interpolation - effectively excluding values that are flagged
- with a flag in to_drop from the interpolation process. Default results in flagger.BAD
+ with a flag in to_drop from the interpolation process. Default results in BAD
values being dropped initially.
Returns
@@ -169,7 +170,7 @@ def linear(
data, flagger = copy(data, field, flagger, field + '_original')
data, flagger = interpolateIndex(
- data, field, flagger, freq, "time", to_drop=to_drop, empty_intervals_flag=flagger.UNFLAGGED, **kwargs
+ data, field, flagger, freq, "time", to_drop=to_drop, empty_intervals_flag=UNFLAGGED, **kwargs
)
return data, flagger
@@ -204,7 +205,7 @@ def interpolate(
Note, that the data only gets interpolated at those (regular) timestamps, that have a valid (existing and
not-na) datapoint preceeding them and one succeeding them within freq range.
Regular timestamp that do not suffice this condition get nan assigned AND The associated flag will be of value
- ``flagger.UNFLAGGED``.
+ ``UNFLAGGED``.
Parameters
----------
@@ -224,7 +225,7 @@ def interpolate(
order.
to_drop : {List[str], str}, default None
Flagtypes you want to drop before interpolation - effectively excluding values that are flagged
- with a flag in `to_drop` from the interpolation process. Default results in ``flagger.BAD``
+ with a flag in `to_drop` from the interpolation process. Default results in ``BAD``
values being dropped initially.
Returns
@@ -246,7 +247,7 @@ def interpolate(
method=method,
inter_order=order,
to_drop=to_drop,
- empty_intervals_flag=flagger.UNFLAGGED,
+ empty_intervals_flag=UNFLAGGED,
**kwargs,
)
return data, flagger
@@ -313,7 +314,7 @@ def mapToOriginal(
details.
to_drop : {List[str], str}, default None
Flagtypes you want to drop before interpolation - effectively excluding values that are flagged
- with a flag in to_drop from the interpolation process. Default results in flagger.BAD
+ with a flag in to_drop from the interpolation process. Default results in BAD
values being dropped initially.
Returns
@@ -393,10 +394,10 @@ def _shift(
description for more details.
empty_intervals_flag : {None, str}, default None
A Flag, that you want to assign to grid points, where no values are avaible to be shifted to.
- Default triggers flagger.UNFLAGGED to be assigned.
+ Default triggers UNFLAGGED to be assigned.
to_drop : {None, str, List[str]}, default None
Flags that refer to values you want to drop before shifting - effectively, excluding values that are flagged
- with a flag in to_drop from the shifting process. Default - to_drop = None - results in flagger.BAD
+ with a flag in to_drop from the shifting process. Default - to_drop = None - results in BAD
values being dropped initially.
freq_check : {None, 'check', 'auto'}, default None
@@ -419,9 +420,9 @@ def _shift(
flagscol = flagger.getFlags(field)
if empty_intervals_flag is None:
- empty_intervals_flag = flagger.UNFLAGGED
+ empty_intervals_flag = UNFLAGGED
- drop_mask = dropper(field, to_drop, flagger, flagger.BAD)
+ drop_mask = dropper(field, to_drop, flagger, BAD)
drop_mask |= datcol.isna()
datcol[drop_mask] = np.nan
datcol.dropna(inplace=True)
@@ -509,18 +510,18 @@ def resample(
containing ONLY nan values, or those containing no values at all, get projected onto nan.
max_invalid_total_f : {None, int}, default None
Same as `max_invalid_total_d`, only applying for the flags. The flag regarded as "invalid" value,
- is the one passed to empty_intervals_flag (default=``flagger.BAD``).
+ is the one passed to empty_intervals_flag (default=``BAD``).
Also this is the flag assigned to invalid/empty intervals.
max_invalid_consec_f : {None, int}, default None
Same as `max_invalid_total_f`, only applying onto flags. The flag regarded as "invalid" value, is the one passed
- to empty_intervals_flag (default=flagger.BAD). Also this is the flag assigned to invalid/empty intervals.
+ to empty_intervals_flag (default=BAD). Also this is the flag assigned to invalid/empty intervals.
flag_agg_func : Callable, default: max
The function you want to aggregate the flags with. It should be capable of operating on the flags dtype
(usually ordered categorical).
empty_intervals_flag : {None, str}, default None
A Flag, that you want to assign to invalid intervals. Invalid are those intervals, that contain nan values only,
or no values at all. Furthermore the empty_intervals_flag is the flag, serving as "invalid" identifyer when
- checking for `max_total_invalid_f` and `max_consec_invalid_f patterns`. Default triggers ``flagger.BAD`` to be
+ checking for `max_total_invalid_f` and `max_consec_invalid_f patterns`. Default triggers ``BAD`` to be
assigned.
to_drop : {None, str, List[str]}, default None
Flags that refer to values you want to drop before resampling - effectively excluding values that are flagged
@@ -547,7 +548,7 @@ def resample(
datcol = data[field]
flagscol = flagger.getFlags(field)
if empty_intervals_flag is None:
- empty_intervals_flag = flagger.BAD
+ empty_intervals_flag = BAD
drop_mask = dropper(field, to_drop, flagger, [])
datcol.drop(datcol[drop_mask].index, inplace=True)
@@ -653,7 +654,7 @@ def reindexFlags(
Defaultly (None), the sampling frequency of source is used.
to_drop : {None, str, List[str]}, default None
Flags referring to values that are to drop before flags projection. Relevant only when projecting with an
- inverted shift method. Defaultly flagger.BAD is listed.
+ inverted shift method. Defaultly BAD is listed.
freq_check : {None, 'check', 'auto'}, default None
- None: do not validate frequency-string passed to `freq`
- 'check': estimate frequency and log a warning if estimate miss matchs frequency string passed to 'freq', or
@@ -720,7 +721,7 @@ def reindexFlags(
#
# starting with the dropping and its memorization:
- drop_mask = dropper(field, to_drop, flagger, flagger.BAD)
+ drop_mask = dropper(field, to_drop, flagger, BAD)
drop_mask |= target_datcol.isna()
target_flagscol_drops = target_flagscol[drop_mask]
target_flagscol.drop(drop_mask[drop_mask].index, inplace=True)
diff --git a/saqc/funcs/scores.py b/saqc/funcs/scores.py
index e723a1a91..90e65ec14 100644
--- a/saqc/funcs/scores.py
+++ b/saqc/funcs/scores.py
@@ -158,7 +158,7 @@ def assignKNNScore(
score_flagger = flagger.initFlags(score_ser)
- if target_field in flagger._flags.columns:
+ if target_field in flagger.columns:
flagger = flagger.slice(drop=target_field)
flagger = flagger.merge(score_flagger)
diff --git a/saqc/funcs/tools.py b/saqc/funcs/tools.py
index 4a900390c..9ad92f94b 100644
--- a/saqc/funcs/tools.py
+++ b/saqc/funcs/tools.py
@@ -40,7 +40,7 @@ def copy(data: DictOfSeries, field: str, flagger: Flagger, new_field: str, **kwa
Flags shape may have changed relatively to the flagger input.
"""
- if new_field in flagger.flags.columns.union(data.columns):
+ if new_field in flagger.columns.union(data.columns):
raise ValueError(f"{field}: field already exist")
flags, extras = flagger.getFlags(field, full=True)
diff --git a/saqc/lib/plotting.py b/saqc/lib/plotting.py
index 1d3883fc5..34b0eb7c4 100644
--- a/saqc/lib/plotting.py
+++ b/saqc/lib/plotting.py
@@ -8,6 +8,7 @@ import pandas as pd
import dios
import matplotlib.pyplot as plt
from typing import List, Dict, Optional
+from saqc.common import *
from saqc.flagger import Flagger
@@ -71,7 +72,7 @@ def plotAllHook(
data, flagger, targets=None, show_info_table: bool = True, annotations: Optional[dios.DictOfSeries] = None,
):
__importHelper()
- targets = flagger.flags.columns if targets is None else targets
+ targets = flagger.columns if targets is None else targets
_plotMultipleVariables(
data_old=None,
flagger_old=None,
@@ -553,11 +554,11 @@ def _splitByFlag(flags: pd.Series, flagger, var: str):
"""
n = flags.isna()
loc = flags.dropna().index
- g = flagger.isFlagged(field=var, loc=loc, flag=flagger.GOOD, comparator="==")
- b = flagger.isFlagged(field=var, loc=loc, flag=flagger.BAD, comparator="==")
- u = flagger.isFlagged(field=var, loc=loc, flag=flagger.UNFLAGGED, comparator="==")
- s = flagger.isFlagged(field=var, loc=loc, flag=flagger.BAD, comparator="<")
- s = flagger.isFlagged(field=var, loc=loc, flag=flagger.GOOD, comparator=">") & s
+ g = flagger.isFlagged(field=var, loc=loc, flag=GOOD, comparator="==")
+ b = flagger.isFlagged(field=var, loc=loc, flag=BAD, comparator="==")
+ u = flagger.isFlagged(field=var, loc=loc, flag=UNFLAGGED, comparator="==")
+ s = flagger.isFlagged(field=var, loc=loc, flag=BAD, comparator="<")
+ s = flagger.isFlagged(field=var, loc=loc, flag=GOOD, comparator=">") & s
return g[g], s[s], b[b], u[u], n[n]
diff --git a/saqc/lib/ts_operators.py b/saqc/lib/ts_operators.py
index 5ae1ca7d0..41001d930 100644
--- a/saqc/lib/ts_operators.py
+++ b/saqc/lib/ts_operators.py
@@ -306,7 +306,7 @@ def aggregate2Freq(
# In the following, we check for empty intervals outside resample.apply, because:
# - resample AND groupBy do insert value zero for empty intervals if resampling with any kind of "sum" application -
# we want "fill_value" to be inserted
- # - we are aggregating data and flags with this function and empty intervals usually would get assigned flagger.BAD
+ # - we are aggregating data and flags with this function and empty intervals usually would get assigned BAD
# flag (where resample inserts np.nan or 0)
data_resampler = data.resample(f"{seconds_total:.0f}s", base=base, closed=closed, label=label)
--
GitLab
From 1ec54cbee35753b6598b4aae4eba44d638dc1675 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 26 Feb 2021 16:34:38 +0100
Subject: [PATCH 004/180] a snake is trampled by a camel
---
saqc/core/core.py | 4 ++--
saqc/flagger/__init__.py | 2 +-
saqc/flagger/flags.py | 18 +++++++++---------
test/flagger/test_flags.py | 6 +++---
4 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index c843617da..1f1ece4da 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -20,7 +20,7 @@ import timeit
import inspect
from saqc.common import *
-from saqc.flagger import init_flags_like, Flagger
+from saqc.flagger import initFlagsLike, Flagger
from saqc.core.lib import APIController, ColumnSelector
from saqc.core.register import FUNC_MAP, SaQCFunction
from saqc.core.modules import FuncModules
@@ -123,7 +123,7 @@ class SaQC(FuncModules):
a flags frame or an already initialised flagger are used.
"""
if flagger is None:
- return init_flags_like(data)
+ return initFlagsLike(data)
for c in flagger.columns.union(data.columns):
if c in flagger:
diff --git a/saqc/flagger/__init__.py b/saqc/flagger/__init__.py
index fd3f6f9f1..e5a86852f 100644
--- a/saqc/flagger/__init__.py
+++ b/saqc/flagger/__init__.py
@@ -1,5 +1,5 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from .flags import Flagger, init_flags_like
+from .flags import Flagger, initFlagsLike
from .history import History
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index 15b8a4efc..c463e983d 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -57,8 +57,8 @@ class Flags:
conversion
----------
- make a dios -> flags.to_dios()
- make a df -> flags.to_frame()
+ make a dios -> flags.toDios()
+ make a df -> flags.toFrame()
"""
def __init__(self, raw_data: Optional[Union[DictLike, Flags]] = None, copy: bool = False):
@@ -70,7 +70,7 @@ class Flags:
raw_data = raw_data._data
# with python 3.7 dicts are insertion-ordered by default
- self._data = self._init_from_raw(raw_data, copy)
+ self._data = self._initFromRaw(raw_data, copy)
# this is a simple cache that reduce the calculation of the flags
# from the entire history of a flag column. The _cache is filled
@@ -79,7 +79,7 @@ class Flags:
# have to much trouble.
self._cache = {}
- def _init_from_raw(self, data, copy) -> Dict[str, History]:
+ def _initFromRaw(self, data, copy) -> Dict[str, History]:
"""
init from dict-like: keys are flag column, values become
initial columns of history(s).
@@ -245,7 +245,7 @@ class Flags:
# ----------------------------------------------------------------------
# transformation and representation
- def to_dios(self) -> dios.DictOfSeries:
+ def toDios(self) -> dios.DictOfSeries:
di = dios.DictOfSeries(columns=self.columns)
for k, v in self._data.items():
@@ -253,14 +253,14 @@ class Flags:
return di.copy()
- def to_frame(self) -> pd.DataFrame:
- return self.to_dios().to_df()
+ def toFrame(self) -> pd.DataFrame:
+ return self.toDios().to_df()
def __repr__(self) -> str:
- return str(self.to_dios()).replace('DictOfSeries', type(self).__name__)
+ return str(self.toDios()).replace('DictOfSeries', type(self).__name__)
-def init_flags_like(reference: Union[pd.Series, DictLike, Flags], initial_value: float = UNFLAGGED) -> Flags:
+def initFlagsLike(reference: Union[pd.Series, DictLike, Flags], initial_value: float = UNFLAGGED) -> Flags:
"""
Create empty Flags, from an reference data structure.
diff --git a/test/flagger/test_flags.py b/test/flagger/test_flags.py
index 999f3f29d..e4f2d8f3e 100644
--- a/test/flagger/test_flags.py
+++ b/test/flagger/test_flags.py
@@ -181,7 +181,7 @@ def test_cache():
assert 'a' not in flags._cache
# cache all
- flags.to_dios()
+ flags.toDios()
for c in flags.columns:
assert c in flags._cache
@@ -202,7 +202,7 @@ def _validate_flags_equals_frame(flags, df):
@pytest.mark.parametrize('data', data)
def test_to_dios(data: np.array):
flags = Flags(data)
- df = flags.to_dios()
+ df = flags.toDios()
assert isinstance(df, dios.DictOfSeries)
_validate_flags_equals_frame(flags, df)
@@ -211,7 +211,7 @@ def test_to_dios(data: np.array):
@pytest.mark.parametrize('data', data)
def test_to_frame(data: np.array):
flags = Flags(data)
- df = flags.to_frame()
+ df = flags.toFrame()
assert isinstance(df, pd.DataFrame)
_validate_flags_equals_frame(flags, df)
--
GitLab
From 3e919778b1d204699fb0ec5708e34d811742b884 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Sun, 28 Feb 2021 12:47:19 +0100
Subject: [PATCH 005/180] fixed "old" masking, adjusted flagRange() as first
test-function
---
saqc/__init__.py | 1 +
saqc/common.py | 4 ++-
saqc/core/core.py | 70 ++++++++++++++++++++++++++++++------------
saqc/funcs/outliers.py | 9 ++++--
4 files changed, 61 insertions(+), 23 deletions(-)
diff --git a/saqc/__init__.py b/saqc/__init__.py
index ddc4f2f02..97de09a39 100644
--- a/saqc/__init__.py
+++ b/saqc/__init__.py
@@ -4,5 +4,6 @@
__version__ = "1.4"
from saqc.core.core import SaQC
+from saqc.common import *
from saqc.flagger import *
from saqc.core.register import register
diff --git a/saqc/common.py b/saqc/common.py
index ce342c917..21010a0c6 100644
--- a/saqc/common.py
+++ b/saqc/common.py
@@ -4,7 +4,9 @@ from numpy import nan as _nan
UNTOUCHED = _nan
UNFLAGGED = 0
-GOOD = 0
DOUBTFUL = 25
BAD = 255
+# aliases
+GOOD = UNFLAGGED
+DOUBT = DOUBTFUL
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 1f1ece4da..749cfcf61 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -77,7 +77,7 @@ def _prepInput(data, flags):
if isinstance(flags, (dios.DictOfSeries, pd.DataFrame, Flagger)):
# NOTE: only test common columns, data as well as flags could
# have more columns than the respective other.
- cols = flags.columns & data.columns
+ cols = flags.columns.intersection(data.columns)
for c in cols:
if not flags[c].index.equals(data[c].index):
raise ValueError(f"the index of 'flags' and 'data' missmatch in column {c}")
@@ -125,13 +125,21 @@ class SaQC(FuncModules):
if flagger is None:
return initFlagsLike(data)
- for c in flagger.columns.union(data.columns):
- if c in flagger:
- continue
- if c in data:
- flagger[c] = pd.Series(UNFLAGGED, index=data[c].index, dtype=float)
+ # add columns that are present in data but not in flagger
+ for c in data.columns.difference(flagger.columns):
+ flagger[c] = pd.Series(UNFLAGGED, index=data[c].index, dtype=float)
+
return flagger
+ def _constructSimple(self) -> SaQC:
+ return SaQC(
+ data=dios.DictOfSeries(),
+ flags=Flagger(),
+ nodata=self._nodata,
+ to_mask=self._to_mask,
+ error_policy=self._error_policy
+ )
+
def readConfig(self, fname):
from saqc.core.reader import readConfig
out = stdcopy.deepcopy(self)
@@ -198,7 +206,7 @@ class SaQC(FuncModules):
# This is way faster for big datasets, than to throw everything in the constructor.
# Simply because of _initFlagger -> merge() -> mergeDios() over all columns.
- new = SaQC(SimpleFlagger(), dios.DictOfSeries(), nodata=self._nodata, error_policy=self._error_policy)
+ new = self._constructSimple()
new._flagger, new._data = flagger, data
return new
@@ -288,7 +296,7 @@ def _saqcCallFunc(locator, controller, function, data, flagger):
# decorated by `register(masking='none')`, and so `to_mask` is ignored.
if masking == 'none' and to_mask not in (None, []):
logging.warning("`to_mask` is given, but the test ignore masking. Please refer to the documentation: TODO")
- to_mask = BAD if to_mask is None else to_mask
+ to_mask = [BAD] if to_mask is None else to_mask
data_in, mask = _maskData(data, flagger, columns, to_mask)
data_result, flagger_result = function(data_in, field, flagger)
@@ -301,22 +309,45 @@ def _saqcCallFunc(locator, controller, function, data, flagger):
return data_result, flagger_result
+# todo: solve with outcome of #GL160
+def _getMask(flags: Union[np.array, pd.Series], to_mask: list) -> Union[np.array, pd.Series]:
+ """
+ Return a mask of flags accordingly to `to_mask`.
+ Return type is same as flags.
+ """
+
+ if isinstance(flags, pd.Series):
+ mask = pd.Series(False, index=flags.index, dtype=bool)
+ else:
+ mask = np.zeros_like(flags, dtype=bool)
+
+ for f in to_mask:
+ mask |= flags == f
+
+ return mask
+
+
+# TODO: this is heavily undertested
def _maskData(data, flagger, columns, to_mask):
- # TODO: this is heavily undertested
- mask = flagger.isFlagged(field=columns, flag=to_mask, comparator='==')
+ # we use numpy here because it is faster
+ mask = dios.DictOfSeries(columns=columns)
data = data.copy()
+
for c in columns:
- col_mask = mask[c].values
+ col_mask = _getMask(flagger[c].to_numpy(), to_mask)
+
if np.any(col_mask):
- col_data = data[c].values.astype(np.float64)
+ col_data = data[c].to_numpy(dtype=np.float64)
col_data[col_mask] = np.nan
+
data[c] = col_data
+ mask[c] = pd.Series(col_mask, index=data[c].index, dtype=bool)
+
return data, mask
+# TODO: this is heavily undertested
def _unmaskData(data_old, mask_old, data_new, flagger_new, to_mask):
- # TODO: this is heavily undertested
-
# NOTE:
# we only need to respect columns, that were masked,
# and are also still present in new data.
@@ -324,22 +355,21 @@ def _unmaskData(data_old, mask_old, data_new, flagger_new, to_mask):
# - any newly assigned columns
# - columns that were excluded from masking
columns = mask_old.dropempty().columns.intersection(data_new.dropempty().columns)
- mask_new = flagger_new.isFlagged(field=columns, flag=to_mask, comparator="==")
for col in columns:
was_masked = mask_old[col]
- is_masked = mask_new[col]
+ is_masked = _getMask(flagger_new[col], to_mask)
# if index changed we just go with the new data.
# A test should use `register(masking='none')` if it changes
# the index but, does not want to have all NaNs on flagged locations.
if was_masked.index.equals(is_masked.index):
- mask = was_masked.values & is_masked.values & data_new[col].isna().values
+ mask = was_masked.to_numpy() & is_masked.to_numpy() & data_new[col].isna().to_numpy()
# reapplying old values on masked positions
if np.any(mask):
- data = np.where(mask, data_old[col].values, data_new[col].values)
- data_new[col] = pd.Series(data=data, index=is_masked.index)
+ data = np.where(mask, data_old[col].to_numpy(), data_new[col].to_numpy())
+ data_new[col] = pd.Series(data=data, index=is_masked.index, dtype=data_old[col].dtype)
return data_new
@@ -367,7 +397,7 @@ def _warnForUnusedKwargs(func, flagger):
# we need to ignore kwargs that are injected or
# used to control the flagger
- ignore = flagger.signature + ("nodata", "func_name")
+ ignore = ("nodata", "func_name", "force", "flag")
missing = []
for kw in func.keywords:
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index a68d5f69c..0971a069b 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -14,6 +14,7 @@ from outliers import smirnov_grubbs
from dios import DictOfSeries
+from saqc.common import *
from saqc.core.register import register
from saqc.flagger import Flagger
from saqc.lib.tools import (
@@ -957,10 +958,14 @@ def flagRange(
Flags values may have changed relatively to the flagger input.
"""
- # using .values is very much faster
+ # using .values is much faster
datacol = data[field].values
mask = (datacol < min) | (datacol > max)
- flagger = flagger.setFlags(field, mask, **kwargs)
+
+ # todo GL162
+ flags = pd.Series(UNTOUCHED, index=data[field].index, dtype=float)
+ flags.loc[mask] = kwargs['flag'] # todo GL161
+ flagger[field] = flags
return data, flagger
--
GitLab
From baf5e11b44ed660f55d11718bff6368a7f869105 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Sun, 28 Feb 2021 14:34:33 +0100
Subject: [PATCH 006/180] added row-select support on set flags
---
saqc/flagger/flags.py | 38 ++++++++++++++++++++++++++------
test/flagger/test_flags.py | 42 ++++++++++++++++++++++++++++++++----
test/flagger/test_history.py | 2 --
3 files changed, 70 insertions(+), 12 deletions(-)
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index c463e983d..081c112d5 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -6,7 +6,7 @@ import dios
from saqc.common import *
from saqc.flagger.history import History
import pandas as pd
-from typing import Union, Dict, DefaultDict, Optional, Type
+from typing import Union, Dict, DefaultDict, Optional, Type, Tuple, Iterable
_VAL = Union[pd.Series, History]
DictLike = Union[
@@ -16,6 +16,13 @@ DictLike = Union[
DefaultDict[str, _VAL],
]
+_Field = str
+SelectT = Union[
+ _Field,
+ Tuple[pd.Series, _Field]
+]
+ValueT = Union[pd.Series, Iterable, float]
+
class _HistAccess:
@@ -163,18 +170,38 @@ class Flags:
return self._cache[key].copy()
- def __setitem__(self, key: str, value: pd.Series, force=False):
+ def __setitem__(self, key: SelectT, value: ValueT, force=False):
# force-KW is internal available only
+ if isinstance(key, tuple):
+ if len(key) != 2:
+ raise KeyError("a single 'column' or a tuple of 'mask, column' must be passt")
+ mask, key = key
+
+ # raises (correct) KeyError
+ tmp = pd.Series(UNTOUCHED, index=self._data[key].index, dtype=float)
+ try:
+ tmp[mask] = value
+ except Exception:
+ raise ValueError('bad mask')
+ else:
+ value = tmp
+
+ # technically it would be possible to select a field and set
+ # the entire column to a scalar flag value (float), but it has
+ # a high potential, that this is not intended by the user.
+ if not isinstance(value, pd.Series):
+ raise ValueError("must pass value of type pd.Series")
+
# if nothing happens no-one writes the history books
- if isinstance(value, pd.Series) and len(value) == 0:
+ if len(value) == 0:
return
if key not in self._data:
self._data[key] = History()
-
+
self._data[key].append(value, force=force)
- self._cache.pop(key, None)
+ self._cache.pop(key, None)
def force(self, key: str, value: pd.Series) -> Flags:
"""
@@ -314,4 +341,3 @@ def initFlagsLike(reference: Union[pd.Series, DictLike, Flags], initial_value: f
# for now we keep this name
Flagger = Flags
-
diff --git a/test/flagger/test_flags.py b/test/flagger/test_flags.py
index e4f2d8f3e..61acdaa46 100644
--- a/test/flagger/test_flags.py
+++ b/test/flagger/test_flags.py
@@ -3,8 +3,8 @@ import dios
import pytest
import numpy as np
import pandas as pd
-from pandas.api.types import is_bool_dtype
-from test.common import TESTFLAGGER, initData
+
+from saqc import BAD, UNFLAGGED
from test.flagger.test_history import (
History,
is_equal as hist_equal,
@@ -156,8 +156,42 @@ def test_set_flags_and_force(data: np.array):
@pytest.mark.parametrize('data', data)
-def test_force_flags(data: np.array):
- pass
+def test_set_flags_with_mask(data: np.array):
+ flags = Flags(data)
+
+ for c in flags.columns:
+ var = flags[c]
+ mask = var == UNFLAGGED
+
+ scalar = 222.
+ flags[mask, c] = scalar
+ assert all(flags[c].loc[mask] == 222.)
+ assert all(flags[c].loc[~mask] != 222.)
+
+ # scalar without mask is not allowed, because
+ # it holds to much potential to set the whole
+ # column unintentionally.
+ with pytest.raises(ValueError):
+ flags[c] = 888.
+
+ vector = var.copy()
+ vector[:] = 333.
+ flags[mask, c] = vector
+ assert all(flags[c].loc[mask] == 333.)
+ assert all(flags[c].loc[~mask] != 333.)
+
+ # works with any that pandas eat, eg with numpy
+ vector[:] = 444.
+ vector = vector.to_numpy()
+ flags[mask, c] = vector
+ assert all(flags[c].loc[mask] == 444.)
+ assert all(flags[c].loc[~mask] != 444.)
+
+ # test length miss-match
+ if len(vector):
+ vector = vector[:-1]
+ with pytest.raises(ValueError):
+ flags[mask, c] = vector
def test_cache():
diff --git a/test/flagger/test_history.py b/test/flagger/test_history.py
index 59261b5f5..e1957615d 100644
--- a/test/flagger/test_history.py
+++ b/test/flagger/test_history.py
@@ -3,8 +3,6 @@
import pytest
import numpy as np
import pandas as pd
-from pandas.api.types import is_bool_dtype
-from test.common import TESTFLAGGER, initData
from saqc.flagger.history import History
# see #GH143 combined backtrack
--
GitLab
From 6fb9d88310b324d1f669e4a4cb9ff0680c35bdd4 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Sun, 28 Feb 2021 14:37:14 +0100
Subject: [PATCH 007/180] fixed flagRange
---
saqc/funcs/outliers.py | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index 0971a069b..d169db254 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -961,11 +961,7 @@ def flagRange(
# using .values is much faster
datacol = data[field].values
mask = (datacol < min) | (datacol > max)
-
- # todo GL162
- flags = pd.Series(UNTOUCHED, index=data[field].index, dtype=float)
- flags.loc[mask] = kwargs['flag'] # todo GL161
- flagger[field] = flags
+ flagger[mask, field] = kwargs['flag'] # todo GL161
return data, flagger
--
GitLab
From e622c68eec4b9abe2b20c285798dc41bf94f2652 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Sun, 28 Feb 2021 15:31:38 +0100
Subject: [PATCH 008/180] dropped force-support as discussed in #GL156
---
saqc/core/core.py | 2 +-
saqc/flagger/flags.py | 25 ++-----------------------
test/flagger/test_flags.py | 10 ++--------
3 files changed, 5 insertions(+), 32 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 749cfcf61..0394e9c5f 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -397,7 +397,7 @@ def _warnForUnusedKwargs(func, flagger):
# we need to ignore kwargs that are injected or
# used to control the flagger
- ignore = ("nodata", "func_name", "force", "flag")
+ ignore = ("nodata", "func_name", "flag")
missing = []
for kw in func.keywords:
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index 081c112d5..84fc97a78 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -170,7 +170,7 @@ class Flags:
return self._cache[key].copy()
- def __setitem__(self, key: SelectT, value: ValueT, force=False):
+ def __setitem__(self, key: SelectT, value: ValueT):
# force-KW is internal available only
if isinstance(key, tuple):
@@ -200,30 +200,9 @@ class Flags:
if key not in self._data:
self._data[key] = History()
- self._data[key].append(value, force=force)
+ self._data[key].append(value, force=True)
self._cache.pop(key, None)
- def force(self, key: str, value: pd.Series) -> Flags:
- """
- Overwrite existing flags, regardless if they are better
- or worse than the existing flags.
-
- Parameters
- ----------
- key : str
- column name
-
- value : pandas.Series
- A series of float flags to force
-
- Returns
- -------
- Flags
- the same flags object with altered flags, no copy
- """
- self.__setitem__(key, value, force=True)
- return self
-
def __delitem__(self, key):
self._data.pop(key)
self._cache.pop(key, None)
diff --git a/test/flagger/test_flags.py b/test/flagger/test_flags.py
index 61acdaa46..1f68c115b 100644
--- a/test/flagger/test_flags.py
+++ b/test/flagger/test_flags.py
@@ -121,7 +121,7 @@ def test_get_flags(data: np.array):
@pytest.mark.parametrize('data', data)
-def test_set_flags_and_force(data: np.array):
+def test_set_flags(data: np.array):
flags = Flags(data)
for c in flags.columns:
@@ -138,15 +138,9 @@ def test_set_flags_and_force(data: np.array):
new[:] = 8888.
assert all(flags.history[c].max() == 9999.)
- # no overwrite if flag-values are not worse
+ # flags always overwrite former
flags[c] = new
assert len(flags.history[c]) == hlen + 2
- assert all(flags.history[c].max() == 9999.)
- assert all(flags.history[c].max() == flags[c])
-
- # but overwrite with force
- flags.force(c, new)
- assert len(flags.history[c]) == hlen + 3
assert all(flags.history[c].max() == 8888.)
assert all(flags.history[c].max() == flags[c])
--
GitLab
From b34674ae4b929ddbb02c47f5e2d071f48238bd80 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Sun, 28 Feb 2021 16:24:46 +0100
Subject: [PATCH 009/180] prepared moving of masking to register
---
saqc/core/register.py | 83 +++++++++++++++++++++++++++++++++++++++++--
1 file changed, 81 insertions(+), 2 deletions(-)
diff --git a/saqc/core/register.py b/saqc/core/register.py
index f9dc7a0f1..c6ed457ce 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -2,6 +2,7 @@
from typing import Dict, Optional
from typing_extensions import Literal
+from functools import wraps
from saqc.core.lib import SaQCFunction
@@ -10,14 +11,92 @@ from saqc.core.lib import SaQCFunction
# will be filled by calls to register
FUNC_MAP: Dict[str, SaQCFunction] = {}
+MaskingStrT = Literal["all", "field", "none"]
-def register(masking: Literal["all", "field", "none"]="all", module: Optional[str]=None):
+def register(masking: MaskingStrT = "all", module: Optional[str] = None):
+
+ # this is called once on module import
def inner(func):
func_name = func.__name__
if module:
func_name = f"{module}.{func_name}"
FUNC_MAP[func_name] = SaQCFunction(func_name, masking, func)
- return func
+
+ # this is called if a register-decorated function is called,
+ # nevertheless if it is called plain or via `SaQC.func`.
+ @wraps(func)
+ def saqcWrapper(*args, **kwargs):
+ args, kwargs, ctrl = _preCall(func, args, kwargs, masking)
+ result = func(*args, **kwargs)
+ return _postCall(result, ctrl)
+
+ return saqcWrapper
return inner
+
+
+def _preCall(func: callable, args: tuple, kwargs: dict, masking: MaskingStrT):
+ """
+ Handler that runs before any call to a saqc-function.
+
+ This is called before each call to a saqc-function, nevertheless if it is
+ called via the SaQC-interface or plain by importing and direct calling.
+
+ Parameters
+ ----------
+ func : callable
+ the function, which is called after this returns. This is not called here!
+
+ args : tuple
+ args to the function
+
+ kwargs : dict
+ kwargs to the function
+
+ masking : str
+ a string indicating which columns in data need masking
+
+ See Also
+ --------
+ _postCall: runs after a saqc-function call
+
+ Returns
+ -------
+ args: tuple
+ arguments to be passed to the actual call
+ kwargs: dict
+ keyword-arguments to be passed to the actual call
+ ctrl: dict
+ control keyword-arguments passed to `_postCall`
+
+ """
+ ctrl = dict(
+ func=func,
+ args=args,
+ kwargs=kwargs,
+ masking=masking,
+ )
+ return args, kwargs, ctrl
+
+
+def _postCall(result, ctrl: dict):
+ """
+ Handler that runs after any call to a saqc-function.
+
+ This is called after a call to a saqc-function, nevertheless if it was
+ called via the SaQC-interface or plain by importing and direct calling.
+
+ Parameters
+ ----------
+ result : tuple
+ the result from the called function, namely: data and flagger
+ ctrl : dict
+ control keywords from `_preCall`
+
+ Returns
+ -------
+ data: dios.DictOfSeries
+ flagger: saqc.flagger.Flagger
+ """
+ return result
--
GitLab
From cdeb5e23a524a465648c3ac4759e27c296c74401 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Sun, 28 Feb 2021 23:01:44 +0100
Subject: [PATCH 010/180] old masking behavior now on every call, invoked via
register-decorator
---
saqc/core/core.py | 107 +++-------------------------
saqc/core/lib.py | 13 ++--
saqc/core/register.py | 161 ++++++++++++++++++++++++++++++++++++++----
saqc/lib/types.py | 3 +
4 files changed, 166 insertions(+), 118 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 0394e9c5f..e850385ae 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -27,7 +27,6 @@ from saqc.core.modules import FuncModules
from saqc.funcs.tools import copy
from saqc.lib.plotting import plotHook, plotAllHook
-
logger = logging.getLogger("SaQC")
@@ -228,12 +227,13 @@ class SaQC(FuncModules):
return data.to_df(), flagger.toFrame()
def _wrap(self, func: SaQCFunction):
- def inner(field: str, *fargs, target: str=None, regex: bool=False, to_mask=None, plot: bool=False, inplace: bool=False, **fkwargs) -> SaQC:
+
+ def inner(field: str, *fargs, target: str = None, regex: bool = False, plot: bool = False, inplace: bool = False, **fkwargs) -> SaQC:
+
+ fkwargs.setdefault('to_mask', self._to_mask)
control = APIController(
- masking=func.masking,
- to_mask=self._to_mask if to_mask is None else to_mask,
- plot=plot,
+ plot=plot
)
locator = ColumnSelector(
@@ -276,113 +276,27 @@ def _saqcCallFunc(locator, controller, function, data, flagger):
field = locator.field
target = locator.target
- to_mask = controller.to_mask
- masking = controller.masking
if (target != field) and (locator.regex is False):
data, flagger = copy(data, field, flagger, target)
field = target
- if masking == 'all':
- columns = data.columns
- elif masking == 'none':
- columns = []
- elif masking == 'field':
- columns = [field]
- else:
- raise ValueError(f"wrong use of `register(masking={masking})`")
-
- # warn if the user explicitly pass `to_mask=..` to a function that is
- # decorated by `register(masking='none')`, and so `to_mask` is ignored.
- if masking == 'none' and to_mask not in (None, []):
- logging.warning("`to_mask` is given, but the test ignore masking. Please refer to the documentation: TODO")
- to_mask = [BAD] if to_mask is None else to_mask
-
- data_in, mask = _maskData(data, flagger, columns, to_mask)
- data_result, flagger_result = function(data_in, field, flagger)
- data_result = _unmaskData(data, mask, data_result, flagger_result, to_mask)
+ data_result, flagger_result = function(data, field, flagger)
# we check the passed function-kwargs after the actual call, because now "hard" errors would already have been
# raised (Eg. `TypeError: got multiple values for argument 'data'`, when the user pass data=...)
- _warnForUnusedKwargs(function, flagger)
+ _warnForUnusedKwargs(function)
return data_result, flagger_result
-# todo: solve with outcome of #GL160
-def _getMask(flags: Union[np.array, pd.Series], to_mask: list) -> Union[np.array, pd.Series]:
- """
- Return a mask of flags accordingly to `to_mask`.
- Return type is same as flags.
- """
-
- if isinstance(flags, pd.Series):
- mask = pd.Series(False, index=flags.index, dtype=bool)
- else:
- mask = np.zeros_like(flags, dtype=bool)
-
- for f in to_mask:
- mask |= flags == f
-
- return mask
-
-
-# TODO: this is heavily undertested
-def _maskData(data, flagger, columns, to_mask):
- # we use numpy here because it is faster
- mask = dios.DictOfSeries(columns=columns)
- data = data.copy()
-
- for c in columns:
- col_mask = _getMask(flagger[c].to_numpy(), to_mask)
-
- if np.any(col_mask):
- col_data = data[c].to_numpy(dtype=np.float64)
- col_data[col_mask] = np.nan
-
- data[c] = col_data
- mask[c] = pd.Series(col_mask, index=data[c].index, dtype=bool)
-
- return data, mask
-
-
-# TODO: this is heavily undertested
-def _unmaskData(data_old, mask_old, data_new, flagger_new, to_mask):
- # NOTE:
- # we only need to respect columns, that were masked,
- # and are also still present in new data.
- # this throws out:
- # - any newly assigned columns
- # - columns that were excluded from masking
- columns = mask_old.dropempty().columns.intersection(data_new.dropempty().columns)
-
- for col in columns:
- was_masked = mask_old[col]
- is_masked = _getMask(flagger_new[col], to_mask)
-
- # if index changed we just go with the new data.
- # A test should use `register(masking='none')` if it changes
- # the index but, does not want to have all NaNs on flagged locations.
- if was_masked.index.equals(is_masked.index):
- mask = was_masked.to_numpy() & is_masked.to_numpy() & data_new[col].isna().to_numpy()
-
- # reapplying old values on masked positions
- if np.any(mask):
- data = np.where(mask, data_old[col].to_numpy(), data_new[col].to_numpy())
- data_new[col] = pd.Series(data=data, index=is_masked.index, dtype=data_old[col].dtype)
-
- return data_new
-
-
-def _warnForUnusedKwargs(func, flagger):
+def _warnForUnusedKwargs(func):
""" Warn for unused kwargs, passed to a SaQC.function.
Parameters
----------
func: SaqcFunction
Saqc internal data structure that hold all function info.
- flagger: saqc.flagger.Flagger
- Flagger object.
Returns
-------
@@ -395,9 +309,8 @@ def _warnForUnusedKwargs(func, flagger):
"""
sig_kws = inspect.signature(func.func).parameters
- # we need to ignore kwargs that are injected or
- # used to control the flagger
- ignore = ("nodata", "func_name", "flag")
+ # we need to ignore kws that are injected or by default hidden in ``**kwargs``
+ ignore = ("nodata", "func_name", "flag", "to_mask")
missing = []
for kw in func.keywords:
diff --git a/saqc/core/lib.py b/saqc/core/lib.py
index 12c6c83e6..7ced1f9ce 100644
--- a/saqc/core/lib.py
+++ b/saqc/core/lib.py
@@ -14,14 +14,13 @@ class ColumnSelector:
regex: bool
+# todo: this seems obsolete
@dataclass
class APIController:
- masking: Literal["none", "field", "all"]
plot: bool
- to_mask: Any = None # flagger.FLAG constants or a list of those
def errorMessage(self):
- return f"masking: {self.masking}\nto_mask: {self.to_mask}"
+ return ""
@dataclass
@@ -35,17 +34,17 @@ class ConfigController(APIController):
class SaQCFunction:
- def __init__(self, name, masking, function, *args, **keywords):
+ def __init__(self, name, function, *args, **keywords):
self.name = name
- self.masking = masking
self.func = function
self.args = args
self.keywords = keywords
def bind(self, *args, **keywords):
return SaQCFunction(
- self.name, self.masking, self.func,
- *(self.args + args), **{**self.keywords, **keywords}
+ self.name, self.func,
+ *(self.args + args),
+ **{**self.keywords, **keywords}
)
def __call__(self, data, field, flagger, *args, **keywords):
diff --git a/saqc/core/register.py b/saqc/core/register.py
index c6ed457ce..a6d678e91 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -1,10 +1,17 @@
#!/usr/bin/env python
-
-from typing import Dict, Optional
+import logging
+from typing import Dict, Optional, Union, Tuple, List
from typing_extensions import Literal
from functools import wraps
+import dataclasses
+import numpy as np
+import pandas as pd
+import dios
+from saqc.common import *
from saqc.core.lib import SaQCFunction
+from saqc.lib.types import FuncReturnT
+from saqc.flagger.flags import Flagger
# NOTE:
# the global SaQC function store,
@@ -14,6 +21,22 @@ FUNC_MAP: Dict[str, SaQCFunction] = {}
MaskingStrT = Literal["all", "field", "none"]
+@dataclasses.dataclass
+class CallCtrl:
+ func: callable
+
+ data: dios.DictOfSeries
+ field: str
+ flagger: Flagger
+
+ args: tuple
+ kwargs: dict
+
+ masking: str = None
+ to_mask: List[float] = None
+ mask: dios.DictOfSeries = None
+
+
def register(masking: MaskingStrT = "all", module: Optional[str] = None):
# this is called once on module import
@@ -21,7 +44,7 @@ def register(masking: MaskingStrT = "all", module: Optional[str] = None):
func_name = func.__name__
if module:
func_name = f"{module}.{func_name}"
- FUNC_MAP[func_name] = SaQCFunction(func_name, masking, func)
+ FUNC_MAP[func_name] = SaQCFunction(func_name, func)
# this is called if a register-decorated function is called,
# nevertheless if it is called plain or via `SaQC.func`.
@@ -67,20 +90,95 @@ def _preCall(func: callable, args: tuple, kwargs: dict, masking: MaskingStrT):
arguments to be passed to the actual call
kwargs: dict
keyword-arguments to be passed to the actual call
- ctrl: dict
+ ctrl: CallCtrl
control keyword-arguments passed to `_postCall`
"""
- ctrl = dict(
- func=func,
- args=args,
- kwargs=kwargs,
- masking=masking,
- )
+ data, field, flagger, *args = args
+
+ ctrl = CallCtrl(func, data.copy(), field, flagger.copy(), args, kwargs, masking=masking)
+ ctrl.to_mask = _getToMask(ctrl)
+ columns = _getMaskingColumns(ctrl)
+ data, ctrl.mask = _maskData(data, flagger, columns, ctrl.to_mask)
+
+ args = data, field, flagger, *args
return args, kwargs, ctrl
-def _postCall(result, ctrl: dict):
+def _getMaskingColumns(ctrl: CallCtrl):
+ """
+ Raises
+ ------
+ ValueError: if given masking literal is not supported
+ """
+ if ctrl.masking == 'all':
+ return ctrl.data.columns
+ if ctrl.masking == 'none':
+ return pd.Index([])
+ if ctrl.masking == 'field':
+ return pd.Index([ctrl.field])
+
+ raise ValueError(f"wrong use of `register(masking={ctrl.masking})`")
+
+
+def _getToMask(ctrl):
+ to_mask = ctrl.kwargs.setdefault('to_mask', None)
+ _warnForUnusedMasking(ctrl.masking, to_mask)
+
+ if to_mask is None:
+ to_mask = [UNFLAGGED]
+
+ return to_mask
+
+
+def _warnForUnusedMasking(masking, to_mask):
+ # warn if the user explicitly pass `to_mask=..` to a function that is
+ # decorated by `register(masking='none')`, by which `to_mask` is ignored
+ if masking == 'none' and to_mask not in (None, []):
+ # todo: see following message
+ logging.warning("`to_mask` is given, but the test ignore masking. Please refer to the documentation: TODO")
+
+
+# TODO: this is heavily undertested
+def _maskData(data, flagger, columns, to_mask) -> Tuple[dios.DictOfSeries, dios.DictOfSeries]:
+ """
+ Mask data with Nans by flags, according to masking and to_mask.
+ """
+ mask = dios.DictOfSeries(columns=columns)
+
+ # we use numpy here because it is faster
+ for c in columns:
+ col_mask = _getMask(flagger[c].to_numpy(), to_mask)
+
+ if np.any(col_mask):
+ col_data = data[c].to_numpy(dtype=np.float64)
+ col_data[col_mask] = np.nan
+
+ data[c] = col_data
+ mask[c] = pd.Series(col_mask, index=data[c].index, dtype=bool)
+
+ return data, mask
+
+
+# todo: solve with outcome of #GL160
+def _getMask(flags: Union[np.array, pd.Series], to_mask: list) -> Union[np.array, pd.Series]:
+ """
+ Return a mask of flags accordingly to `to_mask`.
+ Return type is same as flags.
+ """
+
+ if isinstance(flags, pd.Series):
+ mask = pd.Series(False, index=flags.index, dtype=bool)
+ else:
+ mask = np.zeros_like(flags, dtype=bool)
+
+ for f in to_mask:
+ mask |= flags == f
+
+ return mask
+
+
+def _postCall(result, ctrl: CallCtrl) -> FuncReturnT:
"""
Handler that runs after any call to a saqc-function.
@@ -96,7 +194,42 @@ def _postCall(result, ctrl: dict):
Returns
-------
- data: dios.DictOfSeries
- flagger: saqc.flagger.Flagger
+ data, flagger : dios.DictOfSeries, saqc.flagger.Flagger
"""
- return result
+ data, flagger = result
+ data = _unmaskData(data_old=ctrl.data, mask_old=ctrl.mask,
+ data_new=data, flagger_new=flagger,
+ to_mask=ctrl.to_mask)
+ return data, flagger
+
+
+# TODO: this is heavily undertested
+def _unmaskData(data_old, mask_old, data_new, flagger_new, to_mask) -> dios.DictOfSeries:
+ # NOTE:
+ # we only need to respect columns, that were masked,
+ # and are also still present in new data.
+ # this throws out:
+ # - any newly assigned columns
+ # - columns that were excluded from masking
+ columns = mask_old.columns.intersection(data_new.columns)
+
+ for col in columns:
+
+ if mask_old[col].empty or data_new[col].empty:
+ continue
+
+ was_masked = mask_old[col]
+ is_masked = _getMask(flagger_new[col], to_mask)
+
+ # if index changed we just go with the new data.
+ # A test should use `register(masking='none')` if it changes
+ # the index but, does not want to have all NaNs on flagged locations.
+ if was_masked.index.equals(is_masked.index):
+ mask = was_masked.to_numpy() & is_masked.to_numpy() & data_new[col].isna().to_numpy()
+
+ # reapplying old values on masked positions
+ if np.any(mask):
+ data = np.where(mask, data_old[col].to_numpy(), data_new[col].to_numpy())
+ data_new[col] = pd.Series(data=data, index=is_masked.index, dtype=data_old[col].dtype)
+
+ return data_new
diff --git a/saqc/lib/types.py b/saqc/lib/types.py
index facebe599..67bddeba6 100644
--- a/saqc/lib/types.py
+++ b/saqc/lib/types.py
@@ -6,8 +6,11 @@ from typing import TypeVar, Union
import numpy as np
import pandas as pd
import dios
+from saqc.flagger.flags import Flagger
T = TypeVar("T")
ArrayLike = TypeVar("ArrayLike", np.ndarray, pd.Series, pd.DataFrame)
PandasLike = TypeVar("PandasLike", pd.Series, pd.DataFrame, dios.DictOfSeries)
DiosLikeT = Union[dios.DictOfSeries, pd.DataFrame]
+
+FuncReturnT = [dios.DictOfSeries, Flagger]
--
GitLab
From 55fb9af8073e56ae466016176f7130233527f8a5 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Mon, 1 Mar 2021 23:09:12 +0100
Subject: [PATCH 011/180] new masking behavior, clear flags before each call
---
saqc/core/register.py | 175 +++++++++++++++++++++++++++---------------
1 file changed, 114 insertions(+), 61 deletions(-)
diff --git a/saqc/core/register.py b/saqc/core/register.py
index a6d678e91..5493deba2 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -11,7 +11,7 @@ import dios
from saqc.common import *
from saqc.core.lib import SaQCFunction
from saqc.lib.types import FuncReturnT
-from saqc.flagger.flags import Flagger
+from saqc.flagger.flags import Flagger, initFlagsLike
# NOTE:
# the global SaQC function store,
@@ -32,29 +32,30 @@ class CallCtrl:
args: tuple
kwargs: dict
- masking: str = None
+ masking: MaskingStrT = None
to_mask: List[float] = None
mask: dios.DictOfSeries = None
def register(masking: MaskingStrT = "all", module: Optional[str] = None):
- # this is called once on module import
+ # executed on module import
def inner(func):
- func_name = func.__name__
- if module:
- func_name = f"{module}.{func_name}"
- FUNC_MAP[func_name] = SaQCFunction(func_name, func)
- # this is called if a register-decorated function is called,
+ # executed if a register-decorated function is called,
# nevertheless if it is called plain or via `SaQC.func`.
@wraps(func)
- def saqcWrapper(*args, **kwargs):
+ def callWrapper(*args, **kwargs):
args, kwargs, ctrl = _preCall(func, args, kwargs, masking)
result = func(*args, **kwargs)
return _postCall(result, ctrl)
- return saqcWrapper
+ func_name = func.__name__
+ if module:
+ func_name = f"{module}.{func_name}"
+ FUNC_MAP[func_name] = SaQCFunction(func_name, callWrapper)
+
+ return callWrapper
return inner
@@ -94,35 +95,65 @@ def _preCall(func: callable, args: tuple, kwargs: dict, masking: MaskingStrT):
control keyword-arguments passed to `_postCall`
"""
+ kwargs.setdefault('to_mask', None)
data, field, flagger, *args = args
ctrl = CallCtrl(func, data.copy(), field, flagger.copy(), args, kwargs, masking=masking)
+
+ # masking
ctrl.to_mask = _getToMask(ctrl)
- columns = _getMaskingColumns(ctrl)
+ columns = _getMaskingColumns(ctrl, ctrl.masking)
data, ctrl.mask = _maskData(data, flagger, columns, ctrl.to_mask)
+ # flags
+ flagger = _prepareFlags(flagger, ctrl)
+
args = data, field, flagger, *args
return args, kwargs, ctrl
-def _getMaskingColumns(ctrl: CallCtrl):
+def _postCall(result, ctrl: CallCtrl) -> FuncReturnT:
+ """
+ Handler that runs after any call to a saqc-function.
+
+ This is called after a call to a saqc-function, nevertheless if it was
+ called via the SaQC-interface or plain by importing and direct calling.
+
+ Parameters
+ ----------
+ result : tuple
+ the result from the called function, namely: data and flagger
+ ctrl : dict
+ control keywords from `_preCall`
+
+ Returns
+ -------
+ data, flagger : dios.DictOfSeries, saqc.flagger.Flagger
+ """
+ data, flagger = result
+ flagger = _restoreFlags(flagger, ctrl)
+ data = _unmaskData(data, ctrl)
+ return data, flagger
+
+
+def _getMaskingColumns(ctrl: CallCtrl, masking: MaskingStrT):
"""
Raises
------
ValueError: if given masking literal is not supported
"""
- if ctrl.masking == 'all':
+ if masking == 'all':
return ctrl.data.columns
- if ctrl.masking == 'none':
+ if masking == 'none':
return pd.Index([])
- if ctrl.masking == 'field':
+ if masking == 'field':
return pd.Index([ctrl.field])
raise ValueError(f"wrong use of `register(masking={ctrl.masking})`")
def _getToMask(ctrl):
- to_mask = ctrl.kwargs.setdefault('to_mask', None)
+ to_mask = ctrl.kwargs['to_mask']
_warnForUnusedMasking(ctrl.masking, to_mask)
if to_mask is None:
@@ -150,7 +181,7 @@ def _maskData(data, flagger, columns, to_mask) -> Tuple[dios.DictOfSeries, dios.
for c in columns:
col_mask = _getMask(flagger[c].to_numpy(), to_mask)
- if np.any(col_mask):
+ if any(col_mask):
col_data = data[c].to_numpy(dtype=np.float64)
col_data[col_mask] = np.nan
@@ -175,61 +206,83 @@ def _getMask(flags: Union[np.array, pd.Series], to_mask: list) -> Union[np.array
for f in to_mask:
mask |= flags == f
- return mask
+ return ~mask
-def _postCall(result, ctrl: CallCtrl) -> FuncReturnT:
+def _prepareFlags(flagger: Flagger, ctrl: CallCtrl) -> Flagger:
"""
- Handler that runs after any call to a saqc-function.
+ Clear flags before each call.
+ """
+ # either the index or the columns itself changed
+ if ctrl.masking == 'none':
+ return flagger
- This is called after a call to a saqc-function, nevertheless if it was
- called via the SaQC-interface or plain by importing and direct calling.
+ return initFlagsLike(flagger, initial_value=UNTOUCHED)
- Parameters
- ----------
- result : tuple
- the result from the called function, namely: data and flagger
- ctrl : dict
- control keywords from `_preCall`
- Returns
- -------
- data, flagger : dios.DictOfSeries, saqc.flagger.Flagger
- """
- data, flagger = result
- data = _unmaskData(data_old=ctrl.data, mask_old=ctrl.mask,
- data_new=data, flagger_new=flagger,
- to_mask=ctrl.to_mask)
- return data, flagger
+def _restoreFlags(flagger: Flagger, ctrl: CallCtrl):
+ if ctrl.masking == 'none':
+ ctrl.flagger = flagger
+
+ else:
+ columns = flagger.columns
+ if ctrl.masking == 'field':
+ columns = columns.difference(ctrl.flagger.columns)
+ columns = columns.append(pd.Index([ctrl.field]))
+
+ for c in columns:
+ ctrl.flagger[c] = flagger[c]
+
+ return ctrl.flagger
# TODO: this is heavily undertested
-def _unmaskData(data_old, mask_old, data_new, flagger_new, to_mask) -> dios.DictOfSeries:
- # NOTE:
- # we only need to respect columns, that were masked,
- # and are also still present in new data.
- # this throws out:
- # - any newly assigned columns
- # - columns that were excluded from masking
- columns = mask_old.columns.intersection(data_new.columns)
-
- for col in columns:
-
- if mask_old[col].empty or data_new[col].empty:
+def _unmaskData(data: dios.DictOfSeries, ctrl: CallCtrl) -> dios.DictOfSeries:
+ """
+ Restore the masked data.
+
+ Notes
+ -----
+ Even if this returns data, it work inplace !
+ """
+ if ctrl.masking == 'none':
+ return data
+
+ # we have two options to implement this:
+ #
+ # =================================
+ # set new data on old
+ # =================================
+ # col in old, in masked, in new:
+ # index differ : old <- new (replace column)
+ # else : old <- new (set on masked: ``old[masked & new.notna()] = new``)
+ # col in new only : old <- new (create column)
+ # col in old only : old (delete column)
+ #
+ #
+ # =================================
+ # set old data on new (implemented)
+ # =================================
+ # col in old, in masked, in new :
+ # index differ : new (keep column)
+ # else : new <- old (set on masked, ``new[masked & new.isna()] = old``)
+ # col in new only : new (keep column)
+ # col in old only : new (ignore, was deleted)
+
+ old = ctrl # this alias simplifies reading a lot
+ columns = old.mask.columns.intersection(data.columns) # in old, in masked, in new
+
+ for c in columns:
+
+ if old.data[c].empty or data[c].empty or old.mask[c].empty:
continue
- was_masked = mask_old[col]
- is_masked = _getMask(flagger_new[col], to_mask)
+ if old.data[c].index.equals(data[c].index):
+ restore_old_val = old.mask[c].to_numpy() & data[c].isna().to_numpy()
- # if index changed we just go with the new data.
- # A test should use `register(masking='none')` if it changes
- # the index but, does not want to have all NaNs on flagged locations.
- if was_masked.index.equals(is_masked.index):
- mask = was_masked.to_numpy() & is_masked.to_numpy() & data_new[col].isna().to_numpy()
+ if any(restore_old_val):
+ ol, nw = old.data[c].to_numpy(), data[c].to_numpy()
+ data.loc[:, c] = np.where(restore_old_val, ol, nw)
- # reapplying old values on masked positions
- if np.any(mask):
- data = np.where(mask, data_old[col].to_numpy(), data_new[col].to_numpy())
- data_new[col] = pd.Series(data=data, index=is_masked.index, dtype=data_old[col].dtype)
+ return data
- return data_new
--
GitLab
From 9d1a02e0a965ab9dcd55fa5f073b6a21cb9fbfca Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Mon, 1 Mar 2021 23:49:11 +0100
Subject: [PATCH 012/180] adjusted `to_mask` according to #GL160, simplified
`unmaskData` and added more comments, because its so hard to wrap the brain
around it, without get fu**..messed up
---
saqc/core/register.py | 43 +++++++++++++++++++++----------------------
1 file changed, 21 insertions(+), 22 deletions(-)
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 5493deba2..fcdcf5049 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -33,7 +33,7 @@ class CallCtrl:
kwargs: dict
masking: MaskingStrT = None
- to_mask: List[float] = None
+ to_mask: float = None
mask: dios.DictOfSeries = None
@@ -157,7 +157,7 @@ def _getToMask(ctrl):
_warnForUnusedMasking(ctrl.masking, to_mask)
if to_mask is None:
- to_mask = [UNFLAGGED]
+ to_mask = UNFLAGGED
return to_mask
@@ -165,9 +165,10 @@ def _getToMask(ctrl):
def _warnForUnusedMasking(masking, to_mask):
# warn if the user explicitly pass `to_mask=..` to a function that is
# decorated by `register(masking='none')`, by which `to_mask` is ignored
- if masking == 'none' and to_mask not in (None, []):
- # todo: see following message
- logging.warning("`to_mask` is given, but the test ignore masking. Please refer to the documentation: TODO")
+ # TODO: fix warning message
+ if masking == 'none' and to_mask not in (None, np.inf):
+ logging.warning("`to_mask` is given, but the saqc-function ignore masking."
+ " Please refer to the documentation: TODO")
# TODO: this is heavily undertested
@@ -191,22 +192,12 @@ def _maskData(data, flagger, columns, to_mask) -> Tuple[dios.DictOfSeries, dios.
return data, mask
-# todo: solve with outcome of #GL160
-def _getMask(flags: Union[np.array, pd.Series], to_mask: list) -> Union[np.array, pd.Series]:
+def _getMask(flags: Union[np.array, pd.Series], to_mask: float) -> Union[np.array, pd.Series]:
"""
Return a mask of flags accordingly to `to_mask`.
Return type is same as flags.
"""
-
- if isinstance(flags, pd.Series):
- mask = pd.Series(False, index=flags.index, dtype=bool)
- else:
- mask = np.zeros_like(flags, dtype=bool)
-
- for f in to_mask:
- mask |= flags == f
-
- return ~mask
+ return flags > to_mask
def _prepareFlags(flagger: Flagger, ctrl: CallCtrl) -> Flagger:
@@ -274,15 +265,23 @@ def _unmaskData(data: dios.DictOfSeries, ctrl: CallCtrl) -> dios.DictOfSeries:
for c in columns:
+ # ignore
if old.data[c].empty or data[c].empty or old.mask[c].empty:
continue
- if old.data[c].index.equals(data[c].index):
- restore_old_val = old.mask[c].to_numpy() & data[c].isna().to_numpy()
+ # on index changed, we simply ignore the old data
+ if not old.data[c].index.equals(data[c].index):
+ continue
+
+ restore_old_val = old.mask[c].to_numpy() & data[c].isna().to_numpy()
+
+ # we have nothing to restore
+ if not any(restore_old_val):
+ continue
- if any(restore_old_val):
- ol, nw = old.data[c].to_numpy(), data[c].to_numpy()
- data.loc[:, c] = np.where(restore_old_val, ol, nw)
+ # restore old values if no new are present
+ ol, nw = old.data[c].to_numpy(), data[c].to_numpy()
+ data.loc[:, c] = np.where(restore_old_val, ol, nw)
return data
--
GitLab
From 6fefd4dd48be0576a332a8aa8d8d7b4f8f074150 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 2 Mar 2021 14:23:01 +0100
Subject: [PATCH 013/180] fixed imports
---
saqc/__init__.py | 3 ++-
saqc/common.py | 15 +++++++++++++--
saqc/core/__init__.py | 2 +-
saqc/flagger/__init__.py | 2 +-
saqc/funcs/changepoints.py | 1 +
saqc/funcs/flagtools.py | 2 +-
saqc/lib/types.py | 15 ++++++++++++++-
7 files changed, 33 insertions(+), 7 deletions(-)
diff --git a/saqc/__init__.py b/saqc/__init__.py
index 97de09a39..0c2bb6d4b 100644
--- a/saqc/__init__.py
+++ b/saqc/__init__.py
@@ -3,7 +3,8 @@
__version__ = "1.4"
-from saqc.core.core import SaQC
+# import order: from small to big
from saqc.common import *
from saqc.flagger import *
from saqc.core.register import register
+from saqc.core.core import SaQC
diff --git a/saqc/common.py b/saqc/common.py
index 21010a0c6..9d4e3d504 100644
--- a/saqc/common.py
+++ b/saqc/common.py
@@ -1,8 +1,17 @@
#!/usr/bin/env python
-from numpy import nan as _nan
+__all__ = [
+ "UNTOUCHED",
+ "UNFLAGGED",
+ "DOUBTFUL",
+ "BAD",
+ "GOOD",
+ "DOUBT",
+]
-UNTOUCHED = _nan
+import numpy as np
+
+UNTOUCHED = np.nan
UNFLAGGED = 0
DOUBTFUL = 25
BAD = 255
@@ -10,3 +19,5 @@ BAD = 255
# aliases
GOOD = UNFLAGGED
DOUBT = DOUBTFUL
+
+
diff --git a/saqc/core/__init__.py b/saqc/core/__init__.py
index 2f4234243..097236acb 100644
--- a/saqc/core/__init__.py
+++ b/saqc/core/__init__.py
@@ -1,5 +1,5 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from saqc.core.core import SaQC, logger
from saqc.core.register import register
+from saqc.core.core import SaQC, logger
diff --git a/saqc/flagger/__init__.py b/saqc/flagger/__init__.py
index e5a86852f..bbf082531 100644
--- a/saqc/flagger/__init__.py
+++ b/saqc/flagger/__init__.py
@@ -1,5 +1,5 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from .flags import Flagger, initFlagsLike
from .history import History
+from .flags import Flagger, initFlagsLike
diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py
index b6e3f8aa4..f7234aa6f 100644
--- a/saqc/funcs/changepoints.py
+++ b/saqc/funcs/changepoints.py
@@ -11,6 +11,7 @@ from typing_extensions import Literal
from dios import DictOfSeries
+from saqc.common import *
from saqc.core.register import register
from saqc.lib.tools import customRoller
from saqc.flagger import Flagger
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index 364256e29..40efed0f3 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -8,11 +8,11 @@ import pandas as pd
from dios import DictOfSeries
from saqc.common import *
+from saqc.lib.types import *
from saqc.core.register import register
from saqc.flagger import Flagger
-
@register(masking='field', module="flagtools")
def clearFlags(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs) -> Tuple[DictOfSeries, Flagger]:
flagger = flagger.clearFlags(field, **kwargs)
diff --git a/saqc/lib/types.py b/saqc/lib/types.py
index 650bfb0e7..9a437a2a9 100644
--- a/saqc/lib/types.py
+++ b/saqc/lib/types.py
@@ -1,5 +1,17 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
+__all__ = [
+ 'T',
+ 'ArrayLike',
+ 'PandasLike',
+ 'DiosLikeT',
+ 'FuncReturnT',
+ 'FreqString',
+ 'ColumnName',
+ 'IntegerWindow',
+ 'TimestampColumnName',
+ 'CurveFitter',
+]
from typing import TypeVar, Union, NewType
from typing_extensions import Protocol, Literal
@@ -7,13 +19,14 @@ from typing_extensions import Protocol, Literal
import numpy as np
import pandas as pd
from dios import DictOfSeries
+from saqc import Flagger
T = TypeVar("T")
ArrayLike = TypeVar("ArrayLike", np.ndarray, pd.Series, pd.DataFrame)
PandasLike = TypeVar("PandasLike", pd.Series, pd.DataFrame, DictOfSeries)
DiosLikeT = Union[DictOfSeries, pd.DataFrame]
-FuncReturnT = [dios.DictOfSeries, Flagger]
+FuncReturnT = [DictOfSeries, Flagger]
# we only support fixed length offsets
FreqString = NewType("FreqString", Literal["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"])
--
GitLab
From a13af8b83d74d2181a1e938d350ac82b8d75c4d3 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 2 Mar 2021 14:35:38 +0100
Subject: [PATCH 014/180] fixed breaks.py
---
saqc/funcs/breaks.py | 26 ++++++++++++--------------
1 file changed, 12 insertions(+), 14 deletions(-)
diff --git a/saqc/funcs/breaks.py b/saqc/funcs/breaks.py
index 7f21609be..107c3c3e7 100644
--- a/saqc/funcs/breaks.py
+++ b/saqc/funcs/breaks.py
@@ -12,6 +12,7 @@ from typing import Tuple
import numpy as np
import pandas as pd
+import pandas.tseries.frequencies
from dios import DictOfSeries
@@ -27,7 +28,7 @@ def flagMissing(
data: DictOfSeries,
field: ColumnName,
flagger: Flagger,
- nodata: float=np.nan,
+ nodata: float = np.nan,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -59,7 +60,7 @@ def flagMissing(
else:
mask = datacol == nodata
- flagger = flagger.setFlags(field, loc=mask, **kwargs)
+ flagger[mask, field] = kwargs['flag']
return data, flagger
@@ -76,7 +77,7 @@ def flagIsolated(
The function flags arbitrary large groups of values, if they are surrounded by sufficiently
large data gaps.
- A gap is a timespan containing either no data or invalid (usually `nan`) and flagged data only.
+ A gap is a timespan containing either no data or data invalid only (usually `nan`) .
Parameters
----------
@@ -85,7 +86,7 @@ def flagIsolated(
field : str
The fieldname of the column, holding the data-to-be-flagged.
flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ A flagger object
gap_window : str
The minimum size of the gap before and after a group of valid values, making this group considered an
isolated group. See condition (2) and (3)
@@ -98,8 +99,7 @@ def flagIsolated(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ The flagger object, holding flags and additional information related to `data`.
Notes
-----
@@ -107,8 +107,8 @@ def flagIsolated(
is considered to be isolated, if:
1. :math:`t_{k+1} - t_n <` `group_window`
- 2. None of the :math:`x_j` with :math:`0 < t_k - t_j <` `gap_window`, is valid or unflagged (preceeding gap).
- 3. None of the :math:`x_j` with :math:`0 < t_j - t_(k+n) <` `gap_window`, is valid or unflagged (succeding gap).
+ 2. None of the :math:`x_j` with :math:`0 < t_k - t_j <` `gap_window`, is valid (preceeding gap).
+ 3. None of the :math:`x_j` with :math:`0 < t_j - t_(k+n) <` `gap_window`, is valid (succeding gap).
See Also
--------
@@ -118,10 +118,9 @@ def flagIsolated(
gap_window = pd.tseries.frequencies.to_offset(gap_window)
group_window = pd.tseries.frequencies.to_offset(group_window)
- col = data[field].mask(flagger.isFlagged(field))
- mask = col.isnull()
+ mask = data[field].isna()
- flags = pd.Series(data=0, index=col.index, dtype=bool)
+ flags = pd.Series(data=0, index=mask.index, dtype=bool)
for srs in groupConsecutives(mask):
if np.all(~srs):
start = srs.index[0]
@@ -133,8 +132,7 @@ def flagIsolated(
if right.all():
flags[start:stop] = True
- flagger = flagger.setFlags(field, flags, **kwargs)
-
+ flagger[mask, field] = kwargs['flag']
return data, flagger
@@ -145,7 +143,7 @@ def flagJumps(
flagger: Flagger,
thresh: float,
winsz: FreqString,
- min_periods: IntegerWindow=1,
+ min_periods: IntegerWindow = 1,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
--
GitLab
From 4ec76257aab7d3639a82869c77c3bdfb7611a8d6 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 2 Mar 2021 14:47:33 +0100
Subject: [PATCH 015/180] fixed changepoints.py
---
saqc/flagger/flags.py | 1 +
saqc/funcs/changepoints.py | 13 +++++++------
2 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index 84fc97a78..c06d8da4e 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -190,6 +190,7 @@ class Flags:
# technically it would be possible to select a field and set
# the entire column to a scalar flag value (float), but it has
# a high potential, that this is not intended by the user.
+ # if desired use ``flagger[:, field] = flag``
if not isinstance(value, pd.Series):
raise ValueError("must pass value of type pd.Series")
diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py
index f7234aa6f..9657b53d0 100644
--- a/saqc/funcs/changepoints.py
+++ b/saqc/funcs/changepoints.py
@@ -30,7 +30,7 @@ def flagChangePoints(
fwd_window: Optional[FreqString]=None,
min_periods_fwd: Optional[IntegerWindow]=None,
closed: Literal["right", "left", "both", "neither"]="both",
- try_to_jit: bool=True,
+ try_to_jit: bool=True, # todo rm
reduce_window: FreqString=None,
reduce_func: Callable[[np.ndarray, np.ndarray], int]=lambda x, _: x.argmax(),
**kwargs
@@ -107,7 +107,7 @@ def assignChangePointCluster(
fwd_window: str=None,
min_periods_fwd: Optional[int]=None,
closed: Literal["right", "left", "both", "neither"]="both",
- try_to_jit: bool=True,
+ try_to_jit: bool=True, # todo: rm
reduce_window: str=None,
reduce_func: Callable[[np.ndarray, np.ndarray], float]=lambda x, _: x.argmax(),
model_by_resids: bool=False,
@@ -205,7 +205,7 @@ def assignChangePointCluster(
stat_func = jit_sf
thresh_func = jit_tf
try_to_jit = True
- except (numba.core.errors.TypingError, numba.core.errors.UnsupportedError, IndexError):
+ except (numba.TypingError, numba.UnsupportedError, IndexError):
try_to_jit = False
logging.warning('Could not jit passed statistic - omitting jitting!')
@@ -219,7 +219,7 @@ def assignChangePointCluster(
residues = pd.Series(np.nan, index=data[field].index)
residues[masked_index] = stat_arr
data[field] = residues
- flagger = flagger.setFlags(field, flag=UNFLAGGED, force=True, **kwargs)
+ flagger[:, field] = UNFLAGGED
return data, flagger
det_index = masked_index[result_arr]
@@ -239,10 +239,11 @@ def assignChangePointCluster(
# (better to start cluster labels with number one)
cluster += 1
data[field] = cluster
- flagger = flagger.setFlags(field, flag=UNFLAGGED, force=True, **kwargs)
+ flagger[:, field] = UNFLAGGED
if flag_changepoints:
- flagger = flagger.setFlags(field, loc=det_index)
+ # todo: does not respect kwargs[flag]
+ flagger[det_index, field] = BAD
return data, flagger
--
GitLab
From fc8ba246f338a2826813acd5d5ce1a380cea85f6 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 2 Mar 2021 15:14:07 +0100
Subject: [PATCH 016/180] added feature, to pass an index as mask to flags,
because it seems a common workflow
---
saqc/flagger/flags.py | 9 ++++++++-
test/flagger/test_flags.py | 28 ++++++++++++++++++++++++++++
2 files changed, 36 insertions(+), 1 deletion(-)
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index c06d8da4e..ee365703c 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -178,8 +178,15 @@ class Flags:
raise KeyError("a single 'column' or a tuple of 'mask, column' must be passt")
mask, key = key
- # raises (correct) KeyError
tmp = pd.Series(UNTOUCHED, index=self._data[key].index, dtype=float)
+
+ # make a mask from an index, because it seems
+ # that passing an index is a very common workflow
+ if isinstance(mask, pd.Index):
+ mask = pd.Series(True, index=mask, dtype=bool)
+ mask = mask.reindex(tmp.index, fill_value=False)
+
+ # raises (correct) KeyError
try:
tmp[mask] = value
except Exception:
diff --git a/test/flagger/test_flags.py b/test/flagger/test_flags.py
index 1f68c115b..83c156011 100644
--- a/test/flagger/test_flags.py
+++ b/test/flagger/test_flags.py
@@ -188,6 +188,34 @@ def test_set_flags_with_mask(data: np.array):
flags[mask, c] = vector
+@pytest.mark.parametrize('data', data)
+def test_set_flags_with_index(data: np.array):
+ flags = Flags(data)
+
+ for c in flags.columns:
+ var = flags[c]
+ mask = var == UNFLAGGED
+ index = mask[mask].index
+
+ scalar = 222.
+ flags[index, c] = scalar
+ assert all(flags[c].loc[mask] == 222.)
+ assert all(flags[c].loc[~mask] != 222.)
+
+ vector = var.copy()
+ vector[:] = 333.
+ flags[index, c] = vector
+ assert all(flags[c].loc[mask] == 333.)
+ assert all(flags[c].loc[~mask] != 333.)
+
+ # works with any that pandas eat, eg with numpy
+ vector[:] = 444.
+ vector = vector.to_numpy()
+ flags[index, c] = vector
+ assert all(flags[c].loc[mask] == 444.)
+ assert all(flags[c].loc[~mask] != 444.)
+
+
def test_cache():
arr = np.array([
[0, 0, 0, 0],
--
GitLab
From ad8703c65be84497e7115b43ef48d05605fadfa9 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 2 Mar 2021 15:22:12 +0100
Subject: [PATCH 017/180] fixed constants.py
---
saqc/funcs/constants.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/saqc/funcs/constants.py b/saqc/funcs/constants.py
index 9e2864ed7..34392fd96 100644
--- a/saqc/funcs/constants.py
+++ b/saqc/funcs/constants.py
@@ -74,7 +74,7 @@ def flagConstants(
m2 = r.max() - r.min() <= thresh
mask = m1 | m2
- flagger = flagger.setFlags(field, mask, **kwargs)
+ flagger[mask, field] = kwargs['flag']
return data, flagger
@@ -150,5 +150,5 @@ def flagByVariance(
# result:
plateaus = (plateaus[plateaus == 1.0]).index
- flagger = flagger.setFlags(field, plateaus, **kwargs)
+ flagger[plateaus, field] = kwargs['flag']
return data, flagger
--
GitLab
From 8eae4b96368d41d34bad9e55e4c55c218f4a3fe8 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 2 Mar 2021 16:03:21 +0100
Subject: [PATCH 018/180] fixed curvefit.py and rolling.py
---
saqc/funcs/curvefit.py | 17 +++++++----------
saqc/funcs/rolling.py | 14 +++++---------
2 files changed, 12 insertions(+), 19 deletions(-)
diff --git a/saqc/funcs/curvefit.py b/saqc/funcs/curvefit.py
index c4862fe87..3a98dfdde 100644
--- a/saqc/funcs/curvefit.py
+++ b/saqc/funcs/curvefit.py
@@ -18,7 +18,6 @@ from saqc.flagger import Flagger
from saqc.lib.ts_operators import polyRollerIrregular, polyRollerNumba, polyRoller, polyRollerNoMissingNumba, polyRollerNoMissing
-
@register(masking='field', module="curvefit")
def fitPolynomial(data: DictOfSeries, field: str, flagger: Flagger,
winsz: Union[int, str],
@@ -103,11 +102,11 @@ def fitPolynomial(data: DictOfSeries, field: str, flagger: Flagger,
Flags values may have changed relatively to the flagger input.
"""
+ # todo: some (rater large) parts are functional similar to saqc.funcs.rolling.roll
if data[field].empty:
return data, flagger
data = data.copy()
to_fit = data[field]
- flags = flagger.getFlags(field)
regular = getFreqDelta(to_fit.index)
if not regular:
if isinstance(winsz, int):
@@ -194,13 +193,11 @@ def fitPolynomial(data: DictOfSeries, field: str, flagger: Flagger,
data[field] = residues
if eval_flags:
- num_cats, codes = flags.factorize()
- num_cats = pd.Series(num_cats, index=flags.index).rolling(winsz, center=True, min_periods=min_periods).max()
- nan_samples = num_cats[num_cats.isna()]
- num_cats.drop(nan_samples.index, inplace=True)
- to_flag = pd.Series(codes[num_cats.astype(int)], index=num_cats.index)
- to_flag = to_flag.align(nan_samples)[0]
- to_flag[nan_samples.index] = flags[nan_samples.index]
- flagger = flagger.setFlags(field, to_flag.values, **kwargs)
+ # with the new flagger we dont have to care
+ # about to set NaNs to the original flags anymore
+ # todo: we does not get any flags here, because of masking=field
+ worst = flagger[field].rolling(winsz, center=True, min_periods=min_periods).max()
+ flagger[field] = worst
return data, flagger
+
diff --git a/saqc/funcs/rolling.py b/saqc/funcs/rolling.py
index ab415bfe0..99f6be681 100644
--- a/saqc/funcs/rolling.py
+++ b/saqc/funcs/rolling.py
@@ -72,7 +72,6 @@ def roll(
data = data.copy()
to_fit = data[field]
- flags = flagger.getFlags(field)
if to_fit.empty:
return data, flagger
@@ -123,13 +122,10 @@ def roll(
data[field] = means
if eval_flags:
- num_cats, codes = flags.factorize()
- num_cats = pd.Series(num_cats, index=flags.index).rolling(winsz, center=True, min_periods=min_periods).max()
- nan_samples = num_cats[num_cats.isna()]
- num_cats.drop(nan_samples.index, inplace=True)
- to_flag = pd.Series(codes[num_cats.astype(int)], index=num_cats.index)
- to_flag = to_flag.align(nan_samples)[0]
- to_flag[nan_samples.index] = flags[nan_samples.index]
- flagger = flagger.setFlags(field, to_flag.values, **kwargs)
+ # with the new flagger we dont have to care
+ # about to set NaNs to the original flags anymore
+ # todo: we does not get any flags here, because of masking=field
+ worst = flagger[field].rolling(winsz, center=True, min_periods=min_periods).max()
+ flagger[field] = worst
return data, flagger
--
GitLab
From 7fa14a1325081392d73dcfd51275be16c9693246 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 2 Mar 2021 21:54:09 +0100
Subject: [PATCH 019/180] fixed drift.py
---
saqc/funcs/drift.py | 83 ++++++++++++++++++++++++++++-----------------
1 file changed, 51 insertions(+), 32 deletions(-)
diff --git a/saqc/funcs/drift.py b/saqc/funcs/drift.py
index fd7d0dcb9..173f3ce51 100644
--- a/saqc/funcs/drift.py
+++ b/saqc/funcs/drift.py
@@ -126,14 +126,17 @@ def flagDriftFromNorm(
data_to_flag = data[fields].to_df()
data_to_flag.dropna(inplace=True)
+
segments = data_to_flag.groupby(pd.Grouper(freq=segment_freq))
for segment in segments:
+
if segment[1].shape[0] <= 1:
continue
+
drifters = detectDeviants(segment[1], metric, norm_spread, norm_frac, linkage_method, 'variables')
for var in drifters:
- flagger = flagger.setFlags(fields[var], loc=segment[1].index, **kwargs)
+ flagger[segment[1].index, fields[var]] = kwargs['flag']
return data, flagger
@@ -193,20 +196,24 @@ def flagDriftFromReference(
data_to_flag = data[fields].to_df()
data_to_flag.dropna(inplace=True)
+
fields = list(fields)
if field not in fields:
fields.append(field)
+
var_num = len(fields)
- segments = data_to_flag.groupby(pd.Grouper(freq=segment_freq))
+ segments = data_to_flag.groupby(pd.Grouper(freq=segment_freq))
for segment in segments:
if segment[1].shape[0] <= 1:
continue
+
for i in range(var_num):
dist = metric(segment[1].iloc[:, i].values, segment[1].loc[:, field].values)
+
if dist > thresh:
- flagger = flagger.setFlags(fields[i], loc=segment[1].index, **kwargs)
+ flagger[segment[1].index, fields[i]] = kwargs['flag']
return data, flagger
@@ -246,7 +253,7 @@ def flagDriftFromScaledNorm(
A dictionary of pandas.Series, holding all the data.
field : str
A dummy parameter.
- flagger : saqc.flagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional informations related to `data`.
fields_scale1 : str
List of fieldnames in data to be included into the flagging process which are scaled according to scaling
@@ -280,7 +287,7 @@ def flagDriftFromScaledNorm(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the input flagger.
@@ -318,11 +325,14 @@ def flagDriftFromScaledNorm(
segments = dat_to_flag.groupby(pd.Grouper(freq=segment_freq))
for segment in segments:
+
if segment[1].shape[0] <= 1:
continue
+
drifters = detectDeviants(segment[1], metric, norm_spread, norm_frac, linkage_method, 'variables')
+
for var in drifters:
- flagger = flagger.setFlags(fields[var], loc=segment[1].index, **kwargs)
+ flagger[segment[1].index, fields[var]] = kwargs['flag']
return data, flagger
@@ -395,22 +405,25 @@ def correctExponentialDrift(
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
"""
-
# 1: extract fit intervals:
if data[maint_data_field].empty:
return data, flagger
+
data = data.copy()
to_correct = data[field]
maint_data = data[maint_data_field]
- drift_frame = pd.DataFrame({"drift_group": np.nan, to_correct.name: to_correct.values}, index=to_correct.index)
+
+ d = {"drift_group": np.nan, to_correct.name: to_correct.values}
+ drift_frame = pd.DataFrame(d, index=to_correct.index)
# group the drift frame
for k in range(0, maint_data.shape[0] - 1):
# assign group numbers for the timespans in between one maintenance ending and the beginning of the next
# maintenance time itself remains np.nan assigned
drift_frame.loc[maint_data.values[k] : pd.Timestamp(maint_data.index[k + 1]), "drift_group"] = k
- drift_grouper = drift_frame.groupby("drift_group")
+
# define target values for correction
+ drift_grouper = drift_frame.groupby("drift_group")
shift_targets = drift_grouper.aggregate(lambda x: x[:cal_mean].mean()).shift(-1)
for k, group in drift_grouper:
@@ -422,13 +435,13 @@ def correctExponentialDrift(
shiftedData = dataSeries + dataShiftVektor
to_correct[shiftedData.index] = shiftedData
+ data[field] = to_correct
+
if flag_maint_period:
to_flag = drift_frame["drift_group"]
to_flag = to_flag.drop(to_flag[: maint_data.index[0]].index)
- to_flag = to_flag[to_flag.isna()]
- flagger = flagger.setFlags(field, loc=to_flag, **kwargs)
-
- data[field] = to_correct
+ to_flag = to_flag.dropna()
+ flagger[to_flag, field] = kwargs['flag']
return data, flagger
@@ -461,7 +474,7 @@ def correctRegimeAnomaly(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to correct.
- flagger : saqc.flagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
cluster_field : str
A string denoting the field in data, holding the cluster label for the data you want to correct.
@@ -484,7 +497,7 @@ def correctRegimeAnomaly(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
"""
@@ -566,7 +579,7 @@ def correctOffset(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to correct.
- flagger : saqc.flagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
max_mean_jump : float
when searching for changepoints in mean - this is the threshold a mean difference in the
@@ -590,7 +603,7 @@ def correctOffset(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
"""
@@ -674,7 +687,7 @@ def flagRegimeAnomaly(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
cluster_field : str
The name of the column in data, holding the cluster labels for the samples in field. (has to be indexed
@@ -694,17 +707,23 @@ def flagRegimeAnomaly(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional informations related to `data`.
Flags values may have changed, relatively to the flagger input.
"""
- data, flagger = assignRegimeAnomaly(data, field, flagger, cluster_field, norm_spread,
- linkage_method=linkage_method, metric=metric, norm_frac=norm_frac,
- set_cluster=False, set_flags=True, **kwargs)
-
- return data, flagger
+ return assignRegimeAnomaly(
+ data, field, flagger,
+ cluster_field,
+ norm_spread,
+ linkage_method=linkage_method,
+ metric=metric,
+ norm_frac=norm_frac,
+ set_cluster=False,
+ set_flags=True,
+ **kwargs
+ )
@register(masking='all', module="drift")
@@ -744,7 +763,7 @@ def assignRegimeAnomaly(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger
+ flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
cluster_field : str
The name of the column in data, holding the cluster labels for the samples in field. (has to be indexed
@@ -770,25 +789,25 @@ def assignRegimeAnomaly(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger
+ flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional informations related to `data`.
Flags values may have changed, relatively to the flagger input.
"""
- clusterser = data[cluster_field]
- cluster = np.unique(clusterser)
- cluster_dios = DictOfSeries({i: data[field][clusterser == i] for i in cluster})
+ series = data[cluster_field]
+ cluster = np.unique(series)
+ cluster_dios = DictOfSeries({i: data[field][series == i] for i in cluster})
plateaus = detectDeviants(cluster_dios, metric, norm_spread, norm_frac, linkage_method, 'samples')
if set_flags:
for p in plateaus:
- flagger = flagger.setFlags(field, loc=cluster_dios.iloc[:, p].index, **kwargs)
+ flagger[cluster_dios.iloc[:, p].index, field] = kwargs['flags']
if set_cluster:
for p in plateaus:
if cluster[p] > 0:
- clusterser[clusterser == cluster[p]] = -cluster[p]
+ series[series == cluster[p]] = -cluster[p]
- data[cluster_field] = clusterser
+ data[cluster_field] = series
return data, flagger
--
GitLab
From 8781fec98d5b99f63d5a4963a867ffb8c211735b Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 2 Mar 2021 23:03:00 +0100
Subject: [PATCH 020/180] fixed flagtools.py, added missing docstrings
---
saqc/funcs/flagtools.py | 167 +++++++++++++++++++++++++++++-----------
1 file changed, 120 insertions(+), 47 deletions(-)
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index 40efed0f3..c72868927 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -11,64 +11,76 @@ from saqc.common import *
from saqc.lib.types import *
from saqc.core.register import register
from saqc.flagger import Flagger
+import warnings
@register(masking='field', module="flagtools")
-def clearFlags(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs) -> Tuple[DictOfSeries, Flagger]:
- flagger = flagger.clearFlags(field, **kwargs)
- return data, flagger
-
-
-@register(masking='field', module="flagtools")
-def forceFlags(data: DictOfSeries, field: ColumnName, flagger: Flagger, flag: Any, **kwargs) -> Tuple[DictOfSeries, Flagger]:
- flagger = flagger.clearFlags(field).setFlags(field, flag=flag, inplace=True, **kwargs)
- return data, flagger
-
-
-@register(masking='field', module="flagtools")
-def flagDummy(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+def forceFlags(
+ data: DictOfSeries, field: ColumnName, flagger: Flagger, flag: float, **kwargs
+) -> Tuple[DictOfSeries, Flagger]:
"""
- Function does nothing but returning data and flagger.
+ Set whole column to a flag value.
Parameters
----------
- data : dios.DictOfSeries
- A dictionary of pandas.Series, holding all the data.
+ data : DictOfSeries
+ data container
field : str
- The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ columns name that holds the data
+ flagger : Flagger
+ flagger object
+ flag : float
+ flag to set
+ kwargs : dict
+ unused
Returns
-------
- data : dios.DictOfSeries
- A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
+ data : DictOfSeries
+ flagger : Flagger
+
+ See Also
+ --------
+ clearFlags : set whole column to UNFLAGGED
+ flagUnflagged : set flag value at all unflagged positions
"""
+ flagger[:, field] = flag
return data, flagger
-@register(masking='field', module="flagtools")
-def flagForceFail(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs):
+# masking='none' is sufficient because call is redirected
+@register(masking='none', module="flagtools")
+def clearFlags(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
- Function raises a runtime error.
+ Set whole column to UNFLAGGED.
Parameters
----------
- data : dios.DictOfSeries
- A dictionary of pandas.Series, holding all the data.
+ data : DictOfSeries
+ data container
field : str
- The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ columns name that holds the data
+ flagger : Flagger
+ flagger object
+ kwargs : dict
+ unused
+
+ Returns
+ -------
+ data, flagger: DictOfSeries, Flagger
+ See Also
+ --------
+ forceFlags : set whole column to a flag value
+ flagUnflagged : set flag value at all unflagged positions
"""
- raise RuntimeError("Works as expected :D")
+ return forceFlags(data, field, flagger, flag=UNFLAGGED, **kwargs)
@register(masking='field', module="flagtools")
-def flagUnflagged(data: DictOfSeries, field: ColumnName, flagger: Flagger, flag: Optional[Any]=None, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+def flagUnflagged(
+ data: DictOfSeries, field: ColumnName, flagger: Flagger, flag: float, **kwargs
+) -> Tuple[DictOfSeries, Flagger]:
"""
Function sets the GOOD flag to all values flagged better then GOOD.
If there is an entry 'flag' in the kwargs dictionary passed, the
@@ -82,9 +94,10 @@ def flagUnflagged(data: DictOfSeries, field: ColumnName, flagger: Flagger, flag:
The fieldname of the column, holding the data-to-be-flagged.
flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional informations related to `data`.
+ flag : float
+ flag value to set, has NO default
kwargs : Dict
- If kwargs contains 'flag' entry, kwargs['flag] is set, if no entry 'flag' is present,
- 'UNFLAGGED' is set.
+ unused
Returns
-------
@@ -92,15 +105,19 @@ def flagUnflagged(data: DictOfSeries, field: ColumnName, flagger: Flagger, flag:
A dictionary of pandas.Series, holding all the data.
flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
- """
- flag = GOOD if flag is None else flag
- flagger = flagger.setFlags(field, flag=flag, **kwargs)
+ See Also
+ --------
+ clearFlags : set whole column to UNFLAGGED
+ forceFlags : set whole column to a flag value
+ """
+ unflagged = flagger[field].isna() | (flagger[field] == UNFLAGGED)
+ flagger[unflagged, field] = flag
return data, flagger
@register(masking='field', module="flagtools")
-def flagGood(data: DictOfSeries, field: ColumnName, flagger: Flagger, flag: Optional[Any]=None, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+def flagGood(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
Function sets the GOOD flag to all values flagged better then GOOD.
@@ -119,9 +136,9 @@ def flagGood(data: DictOfSeries, field: ColumnName, flagger: Flagger, flag: Opti
A dictionary of pandas.Series, holding all the data.
flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
-
"""
- return flagUnflagged(data, field, flagger, flag=flag, **kwargs)
+ warnings.warn("'flagGood' is deprecated and does nothing, use 'flagUnflagged' instead", DeprecationWarning)
+ return data, flagger
@register(masking='field', module="flagtools")
@@ -165,7 +182,8 @@ def flagManual(
Returns
-------
- data, flagger: original data, modified flagger
+ data : original data
+ flagger : modified flagger
Examples
--------
@@ -181,7 +199,7 @@ def flagManual(
Bear in mind that only exact timestamps apply, any offset will result in ignoring
the timestamp.
>>> _, fl = flagManual(data, field, flagger, mdata, mflag=1, method='ontime')
- >>> fl.isFlagged(field)
+ >>> fl[field] > UNFLAGGED
2000-01-31 False
2000-02-01 True
2000-02-02 False
@@ -194,7 +212,7 @@ def flagManual(
With the 'right-open' method, the mdata is forward fill:
>>> _, fl = flagManual(data, field, flagger, mdata, mflag=1, method='right-open')
- >>> fl.isFlagged(field)
+ >>> fl[field] > UNFLAGGED
2000-01-31 False
2000-02-01 True
2000-02-02 True
@@ -206,7 +224,7 @@ def flagManual(
With the 'left-open' method, backward filling is used:
>>> _, fl = flagManual(data, field, flagger, mdata, mflag=1, method='left-open')
- >>> fl.isFlagged(field)
+ >>> fl[field] > UNFLAGGED
2000-01-31 False
2000-02-01 False
2000-02-02 True
@@ -226,13 +244,19 @@ def flagManual(
raise ValueError("mdata has no index")
if method == "plain":
+
if hasindex:
mdata = mdata.to_numpy()
+
if len(mdata) != len(dat):
raise ValueError("mdata must have same length then data")
+
mdata = pd.Series(mdata, index=dat.index)
+
+ # reindex will do the job later
elif method == "ontime":
- pass # reindex will do the job later
+ pass
+
elif method in ["left-open", "right-open"]:
mdata = mdata.reindex(dat.index.union(mdata.index))
@@ -243,10 +267,59 @@ def flagManual(
# <--t0](<--t1](<-- (bfill)
if method == "left-open":
mdata = mdata.bfill()
+
else:
raise ValueError(method)
mask = mdata == mflag
mask = mask.reindex(dat.index).fillna(False)
- flagger = flagger.setFlags(field=field, loc=mask, **kwargs)
+
+ flagger[mask, field] = kwargs['flag']
+ return data, flagger
+
+
+@register(masking='none', module="flagtools")
+def flagDummy(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+ """
+ Function does nothing but returning data and flagger.
+
+ Parameters
+ ----------
+ data : dios.DictOfSeries
+ A dictionary of pandas.Series, holding all the data.
+ field : str
+ The fieldname of the column, holding the data-to-be-flagged.
+ flagger : saqc.flagger.Flagger
+ A flagger object, holding flags and additional informations related to `data`.
+
+ Returns
+ -------
+ data : dios.DictOfSeries
+ A dictionary of pandas.Series, holding all the data.
+ flagger : saqc.flagger.Flagger
+ The flagger object, holding flags and additional Informations related to `data`.
+ """
return data, flagger
+
+
+@register(masking='none', module="flagtools")
+def flagForceFail(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs):
+ """
+ Function raises a runtime error.
+
+ Parameters
+ ----------
+ data : dios.DictOfSeries
+ A dictionary of pandas.Series, holding all the data.
+ field : str
+ The fieldname of the column, holding the data-to-be-flagged.
+ flagger : saqc.flagger.Flagger
+ A flagger object, holding flags and additional informations related to `data`.
+
+ Raises
+ ------
+ RuntimeError : always
+ """
+ raise RuntimeError("Works as expected :D")
+
+
--
GitLab
From eaad306bb07e701558c69f33c1a77e362fcbf8a3 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 3 Mar 2021 00:04:00 +0100
Subject: [PATCH 021/180] fixed interpolation.py (except the
core-interpolation-routine, because this is a crazy blob of code)
---
saqc/funcs/drift.py | 2 +-
saqc/funcs/interpolation.py | 95 +++++++++++++++++--------------------
saqc/lib/tools.py | 2 +-
3 files changed, 46 insertions(+), 53 deletions(-)
diff --git a/saqc/funcs/drift.py b/saqc/funcs/drift.py
index 173f3ce51..dfb530b04 100644
--- a/saqc/funcs/drift.py
+++ b/saqc/funcs/drift.py
@@ -308,7 +308,7 @@ def flagDriftFromScaledNorm(
for field1 in fields_scale1:
for field2 in fields_scale2:
- slope, intercept, _, _, _ = stats.linregress(data_to_flag[field1], data_to_flag[field2])
+ slope, intercept, *_ = stats.linregress(data_to_flag[field1], data_to_flag[field2])
convert_slope.append(slope)
convert_intercept.append(intercept)
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index 9453a5888..98508c88b 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -1,6 +1,6 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-
+import warnings
from typing import Tuple, Union, Optional, Any, Callable, Sequence
from typing_extensions import Literal
@@ -24,7 +24,7 @@ def interpolateByRolling(
func: Callable[[pd.Series], float]=np.median,
center: bool=True,
min_periods: int=0,
- interpol_flag=Any,
+ flag: float = UNFLAGGED,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -52,9 +52,9 @@ def interpolateByRolling(
min_periods : int
Minimum number of valid (not np.nan) values that have to be available in a window for its aggregation to be
computed.
- interpol_flag : {'GOOD', 'BAD', 'UNFLAGGED', str}, default 'UNFLAGGED'
- Flag that is to be inserted for the interpolated values. You can either pass one of the three major flag-classes
- or specify directly a certain flag from the passed flagger.
+ flag : float, default 0
+ Flag that is to be inserted for the interpolated values. If ``None`` no flags are set.
+ Defaults to ``0`` aka. ``UNFLAGGED``.
Returns
-------
@@ -82,26 +82,24 @@ def interpolateByRolling(
datcol[na_mask] = rolled[na_mask]
data[field] = datcol
- if interpol_flag:
- if interpol_flag in ["BAD", "UNFLAGGED", "GOOD"]:
- interpol_flag = getattr(flagger, interpol_flag)
- flagger = flagger.setFlags(field, loc=interpolated, force=True, flag=interpol_flag, **kwargs)
+ if flag is not None:
+ flagger[interpolated, field] = flag
return data, flagger
@register(masking='field', module="interpolation")
def interpolateInvalid(
- data: DictOfSeries,
- field: str,
- flagger: Flagger,
- method: Literal["linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric", "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"],
- inter_order: int=2,
- inter_limit: int=2,
- interpol_flag: Any="UNFLAGGED",
- downgrade_interpolation: bool=False,
- not_interpol_flags: Optional[Union[Any, Sequence[Any]]]=None,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flagger: Flagger,
+ method: Literal["linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric", "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"],
+ inter_order: int=2,
+ inter_limit: int=2,
+ downgrade_interpolation: bool=False,
+ not_interpol_flags=None,
+ flag: float = UNFLAGGED,
+ **kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -113,9 +111,6 @@ def interpolateInvalid(
Note, that the `inter_limit` keyword really restricts the interpolation to chunks, not containing more than
`inter_limit` successive nan entries.
- Note, that the function differs from ``proc_interpolateGrid``, in its behaviour to ONLY interpolate nan values that
- were already present in the data passed.
-
Parameters
----------
data : dios.DictOfSeries
@@ -132,14 +127,14 @@ def interpolateInvalid(
order.
inter_limit : int, default 2
Maximum number of consecutive 'nan' values allowed for a gap to be interpolated.
- interpol_flag : {'GOOD', 'BAD', 'UNFLAGGED', str}, default 'UNFLAGGED'
- Flag that is to be inserted for the interpolated values. You can either pass one of the three major flag-classes
- or specify directly a certain flag from the passed flagger.
+ flag : float or None, default 0
+ Flag that is to be inserted for the interpolated values. If ``None`` no flags are set.
+ Defaults to ``0`` aka. ``UNFLAGGED``.
downgrade_interpolation : bool, default False
- If interpolation can not be performed at `inter_order` - (not enough values or not implemented at this order) -
- automaticalyy try to interpolate at order `inter_order` :math:`- 1`.
- not_interpol_flags : {None, str, List[str]}, default None
- A list of flags or a single Flag, marking values, you want NOT to be interpolated.
+ If interpolation can not be performed at `inter_order`, because not enough values are present or the order
+ is not implemented for the passed method, automatically try to interpolate at ``inter_order-1``.
+ not_interpol_flags : None
+ deprecated
Returns
-------
@@ -162,19 +157,12 @@ def interpolateInvalid(
)
interpolated = data[field].isna() & inter_data.notna()
- if not_interpol_flags:
- for f in toSequence(not_interpol_flags):
- if f in ["BAD", "UNFLAGGED", "GOOD"]:
- f = getattr(flagger, interpol_flag)
- is_flagged = flagger.isFlagged(flag=f)[field]
- cond = is_flagged & interpolated
- inter_data.mask(cond, np.nan, inplace=True)
- interpolated &= inter_data.notna()
+ # todo: remove with version 2.0
+ if not_interpol_flags is not None:
+ raise ValueError("'not_interpol_flags' is deprecated")
- if interpol_flag:
- if interpol_flag in ["BAD", "UNFLAGGED", "GOOD"]:
- interpol_flag = getattr(flagger, interpol_flag)
- flagger = flagger.setFlags(field, loc=interpolated, force=True, flag=interpol_flag, **kwargs)
+ if flag is not None:
+ flagger[interpolated, field] = flag
data[field] = inter_data
return data, flagger
@@ -193,7 +181,7 @@ def interpolateIndex(
empty_intervals_flag: Any=None,
grid_field: str=None,
inter_limit: int=2,
- freq_check: Optional[Literal["check", "auto"]]=None,
+ freq_check: Optional[Literal["check", "auto"]]=None, # todo: rm not a user decision
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
@@ -261,11 +249,13 @@ def interpolateIndex(
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
+ raise NotImplementedError("currently not available - rewrite needed")
datcol = data[field]
datcol = datcol.copy()
flagscol = flagger.getFlags(field)
freq = evalFreqStr(freq, freq_check, datcol.index)
+
if empty_intervals_flag is None:
empty_intervals_flag = BAD
@@ -274,14 +264,15 @@ def interpolateIndex(
drop_mask |= datcol.isna()
datcol[drop_mask] = np.nan
datcol.dropna(inplace=True)
- freq = evalFreqStr(freq, freq_check, datcol.index)
+
if datcol.empty:
data[field] = datcol
reshaped_flagger = flagger.initFlags(datcol).setFlags(field, flag=flagscol, force=True, inplace=True, **kwargs)
flagger = flagger.slice(drop=field).merge(reshaped_flagger, subset=[field], inplace=True)
return data, flagger
- # account for annoying case of subsequent frequency aligned values, differing exactly by the margin
- # 2*freq:
+
+ # account for annoying case of subsequent frequency aligned values,
+ # which differ exactly by the margin of 2*freq
spec_case_mask = datcol.index.to_series()
spec_case_mask = spec_case_mask - spec_case_mask.shift(1)
spec_case_mask = spec_case_mask == 2 * pd.Timedelta(freq)
@@ -293,24 +284,27 @@ def interpolateIndex(
# prepare grid interpolation:
if grid_field is None:
- grid_index = pd.date_range(start=datcol.index[0].floor(freq), end=datcol.index[-1].ceil(freq), freq=freq,
- name=datcol.index.name)
+ start, end = datcol.index[0].floor(freq), datcol.index[-1].ceil(freq)
+ grid_index = pd.date_range(start=start, end=end, freq=freq, name=datcol.index.name)
else:
grid_index = data[grid_field].index
-
aligned_start = datcol.index[0] == grid_index[0]
aligned_end = datcol.index[-1] == grid_index[-1]
datcol = datcol.reindex(datcol.index.join(grid_index, how="outer",))
# do the interpolation
inter_data, chunk_bounds = interpolateNANs(
- datcol, method, order=inter_order, inter_limit=inter_limit, downgrade_interpolation=downgrade_interpolation,
+ data=datcol,
+ method=method,
+ order=inter_order,
+ inter_limit=inter_limit,
+ downgrade_interpolation=downgrade_interpolation,
return_chunk_bounds=True
)
+ # override falsely interpolated values:
if grid_field is None:
- # override falsely interpolated values:
inter_data[spec_case_mask.index] = np.nan
# store interpolated grid
@@ -352,7 +346,6 @@ def interpolateIndex(
cats_dict = {num: key for (key, num) in cats_dict.items()}
flagscol = flagscol.astype(int, errors="ignore").replace(cats_dict)
flagscol[flagscol.isna()] = empty_intervals_flag
- # ...hack done
# we might miss the flag for interpolated data grids last entry (if we miss it - the datapoint is always nan
# - just settling a convention here(resulting GRID should start BEFORE first valid data entry and range to AFTER
diff --git a/saqc/lib/tools.py b/saqc/lib/tools.py
index 5698e8151..01d534367 100644
--- a/saqc/lib/tools.py
+++ b/saqc/lib/tools.py
@@ -441,7 +441,7 @@ def evalFreqStr(freq, check, index):
if check == 'check':
f_passed_seconds = pd.Timedelta(f_passed).total_seconds()
freq_seconds = pd.Timedelta(freq).total_seconds()
- if (f_passed_seconds != freq_seconds):
+ if f_passed_seconds != freq_seconds:
logging.warning(f"Sampling rate estimate ({freq}) missmatches passed frequency ({f_passed}).")
elif check == 'auto':
if freq is None:
--
GitLab
From 4af3c0bbc5517382105c14a5db53009cec85e7a6 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 3 Mar 2021 01:24:01 +0100
Subject: [PATCH 022/180] fixed outliers.py
---
saqc/flagger/flags.py | 3 +-
saqc/funcs/outliers.py | 208 ++++++++++++++++++++++++-----------------
2 files changed, 124 insertions(+), 87 deletions(-)
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index ee365703c..175833ff5 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -19,7 +19,8 @@ DictLike = Union[
_Field = str
SelectT = Union[
_Field,
- Tuple[pd.Series, _Field]
+ Tuple[pd.Series, _Field],
+ Tuple[pd.Index, _Field],
]
ValueT = Union[pd.Series, Iterable, float]
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index eb42a63bd..22bace68c 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -51,8 +51,8 @@ def flagByStray(
The fieldname of the column, holding the data-to-be-flagged.
flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
- partition_freq : {None, str, int}, default None
- partition_freq : {np.inf, float, str}, default np.inf
+
+ partition_freq : str, int, or None, default None
Determines the segmentation of the data into partitions, the kNN algorithm is
applied onto individually.
@@ -65,10 +65,12 @@ def flagByStray(
Minimum number of periods per partition that have to be present for a valid outlier dettection to be made in
this partition. (Only of effect, if `partition_freq` is an integer.) Partition min value must always be
greater then the nn_neighbors value.
+
iter_start : float, default 0.5
Float in [0,1] that determines which percentage of data is considered "normal". 0.5 results in the stray
algorithm to search only the upper 50 % of the scores for the cut off point. (See reference section for more
information)
+
alpha : float, default 0.05
Level of significance by which it is tested, if a score might be drawn from another distribution, than the
majority of the data.
@@ -78,9 +80,8 @@ def flagByStray(
[1] Talagala, P. D., Hyndman, R. J., & Smith-Miles, K. (2019). Anomaly detection in high dimensional data.
arXiv preprint arXiv:1908.04000.
"""
+ scores = data[field].dropna(inplace=True)
- scores = data[field]
- scores.dropna(inplace=True)
if scores.empty:
return data, flagger
@@ -89,6 +90,7 @@ def flagByStray(
if isinstance(partition_freq, str):
partitions = scores.groupby(pd.Grouper(freq=partition_freq))
+
else:
grouper_series = pd.Series(data=np.arange(0, scores.shape[0]), index=scores.index)
grouper_series = grouper_series.transform(lambda x: int(np.floor(x / partition_freq)))
@@ -96,29 +98,32 @@ def flagByStray(
# calculate flags for every partition
for _, partition in partitions:
+
if partition.empty | (partition.shape[0] < partition_min):
continue
+
sample_size = partition.shape[0]
+
sorted_i = partition.values.argsort()
resids = partition.values[sorted_i]
gaps = np.append(0, np.diff(resids))
+
tail_size = int(max(min(50, np.floor(sample_size / 4)), 2))
tail_indices = np.arange(2, tail_size + 1)
+
i_start = int(max(np.floor(sample_size * iter_start), 1) + 1)
ghat = np.array([np.nan] * sample_size)
+
for i in range(i_start - 1, sample_size):
ghat[i] = sum((tail_indices / (tail_size - 1)) * gaps[i - tail_indices + 1])
log_alpha = np.log(1 / alpha)
- trigger_flagging = False
for iter_index in range(i_start - 1, sample_size):
if gaps[iter_index] > log_alpha * ghat[iter_index]:
- trigger_flagging = True
+ index = partition.index[sorted_i[iter_index:]]
+ flagger[index, field] = kwargs['flag']
break
- if trigger_flagging:
- flagger = flagger.setFlags(field, loc=partition.index[sorted_i[iter_index:]], **kwargs)
-
return data, flagger
@@ -167,59 +172,68 @@ def _evalStrayLabels(
section for more details ([1]).
at_least_one : bool, default True
If none of the variables, the outlier label shall be reduced to, is an outlier with regard
- to the test, all (True) or none (False) of the variables are flagged outliers
+ to the test, all (True) or none (False) of the variables are flagged
References
----------
[1] https://www.itl.nist.gov/div898/handbook/eda/section3/eda35h.htm
"""
-
val_frame = data[fields].to_df()
- stray_detects = flagger.isFlagged()[field]
+ stray_detects = flagger[field] > UNFLAGGED
stray_detects = stray_detects[stray_detects]
to_flag_frame = pd.DataFrame(False, columns=fields, index=stray_detects.index)
- to_flag_index = to_flag_frame.index
+
if reduction_range is None:
for field in to_flag_frame.columns:
- flagger = flagger.setFlags(field, loc=to_flag_index)
+ flagger[to_flag_frame.index, field] = kwargs['flag']
return data, flagger
for var in fields:
- for index in enumerate(to_flag_index):
- index_slice = slice(index[1] - pd.Timedelta(reduction_range), index[1] + pd.Timedelta(reduction_range))
+ for index in enumerate(to_flag_frame.index):
+ index_slice = slice(index[1] - pd.Timedelta(reduction_range), index[1] + pd.Timedelta(reduction_range))
test_slice = val_frame[var][index_slice].dropna()
+
# check, wheather value under test is sufficiently centered:
- first_valid = test_slice.first_valid_index()
- last_valid = test_slice.last_valid_index()
+ first = test_slice.first_valid_index()
+ last = test_slice.last_valid_index()
min_range = pd.Timedelta(reduction_range)/4
- polydeg = 2
- if ((pd.Timedelta(index[1] - first_valid) < min_range) |
- (pd.Timedelta(last_valid - index[1]) < min_range)):
+
+ if pd.Timedelta(index[1] - first) < min_range or pd.Timedelta(last - index[1]) < min_range:
polydeg = 0
- if reduction_drop_flagged:
- test_slice = test_slice.drop(to_flag_index, errors='ignore')
- if test_slice.shape[0] >= reduction_min_periods:
- x = (test_slice.index.values.astype(float))
- x_0 = x[0]
- x = (x - x_0)/10**12
- polyfitted = poly.polyfit(y=test_slice.values, x=x, deg=polydeg)
- testval = poly.polyval((float(index[1].to_numpy()) - x_0)/10**12, polyfitted)
- testval = val_frame[var][index[1]] - testval
- resids = test_slice.values - poly.polyval(x, polyfitted)
- med_resids = np.median(resids)
- MAD = np.median(np.abs(resids - med_resids))
- crit_val = 0.6745 * (abs(med_resids - testval)) / MAD
- if crit_val > reduction_thresh:
- to_flag_frame.loc[index[1], var] = True
else:
+ polydeg = 2
+
+ if reduction_drop_flagged:
+ test_slice = test_slice.drop(to_flag_frame.index, errors='ignore')
+
+ if test_slice.shape[0] < reduction_min_periods:
+ to_flag_frame.loc[index[1], var] = True
+ continue
+
+ x = (test_slice.index.values.astype(float))
+ x_0 = x[0]
+ x = (x - x_0)/10**12
+
+ polyfitted = poly.polyfit(y=test_slice.values, x=x, deg=polydeg)
+
+ testval = poly.polyval((float(index[1].to_numpy()) - x_0)/10**12, polyfitted)
+ testval = val_frame[var][index[1]] - testval
+
+ resids = test_slice.values - poly.polyval(x, polyfitted)
+ med_resids = np.median(resids)
+ MAD = np.median(np.abs(resids - med_resids))
+ crit_val = 0.6745 * (abs(med_resids - testval)) / MAD
+
+ if crit_val > reduction_thresh:
to_flag_frame.loc[index[1], var] = True
if at_least_one:
to_flag_frame[~to_flag_frame.any(axis=1)] = True
for field in to_flag_frame.columns:
- flagger = flagger.setFlags(field, loc=to_flag_frame[field][to_flag_frame[field]].index, **kwargs)
+ col = to_flag_frame[field]
+ flagger[col[col].index, field] = kwargs['flag']
return data, flagger
@@ -457,20 +471,31 @@ def flagMVScores(
outliers. See description of the `threshing` parameter for more details. Although [2] gives a fully detailed
overview over the `stray` algorithm.
"""
- data, flagger = assignKNNScore(data, 'dummy', flagger, fields, n_neighbors=n_neighbors, trafo=trafo,
- trafo_on_partition=trafo_on_partition, scoring_func=scoring_func,
- target_field='kNN_scores', partition_freq=stray_partition,
- kNN_algorithm='ball_tree', partition_min=stray_partition_min, **kwargs)
-
- data, flagger = flagByStray(data, 'kNN_scores', flagger,
- partition_freq=stray_partition,
- partition_min=stray_partition_min,
- iter_start=iter_start,
- alpha=alpha, **kwargs)
+ data, flagger = assignKNNScore(
+ data, 'dummy', flagger,
+ fields=fields,
+ n_neighbors=n_neighbors,
+ trafo=trafo,
+ trafo_on_partition=trafo_on_partition,
+ scoring_func=scoring_func,
+ target_field='kNN_scores',
+ partition_freq=stray_partition,
+ kNN_algorithm='ball_tree',
+ partition_min=stray_partition_min, **kwargs)
+
+ data, flagger = flagByStray(
+ data, 'kNN_scores', flagger,
+ partition_freq=stray_partition,
+ partition_min=stray_partition_min,
+ iter_start=iter_start,
+ alpha=alpha, **kwargs)
data, flagger = _evalStrayLabels(
- data, 'kNN_scores', flagger, fields, reduction_range=reduction_range,
- reduction_drop_flagged=reduction_drop_flagged, reduction_thresh=reduction_thresh,
+ data, 'kNN_scores', flagger,
+ fields=fields,
+ reduction_range=reduction_range,
+ reduction_drop_flagged=reduction_drop_flagged,
+ reduction_thresh=reduction_thresh,
reduction_min_periods=reduction_min_periods, **kwargs)
return data, flagger
@@ -488,7 +513,7 @@ def flagRaise(
mean_raise_factor: float=2.,
min_slope: Optional[float]=None,
min_slope_weight: float=0.8,
- numba_boost: bool=True,
+ numba_boost: bool=True, # todo: rm, not a user decision
**kwargs,
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -637,16 +662,16 @@ def flagRaise(
# check means against critical raise value:
to_flag = dataseries >= weighted_rolling_mean + (raise_series / mean_raise_factor)
to_flag &= raise_series.notna()
- flagger = flagger.setFlags(field, to_flag[to_flag].index, **kwargs)
+ flagger[to_flag[to_flag].index, field] = kwargs['flag']
return data, flagger
@register(masking='field', module="outliers")
-def flagMAD(data: DictOfSeries, field: ColumnName, flagger: Flagger, window: FreqString, z: float=3.5, **kwargs) -> Tuple[DictOfSeries, Flagger]:
-
+def flagMAD(
+ data: DictOfSeries, field: ColumnName, flagger: Flagger, window: FreqString, z: float=3.5, **kwargs
+) -> Tuple[DictOfSeries, Flagger]:
"""
-
The function represents an implementation of the modyfied Z-score outlier detection method.
See references [1] for more details on the algorithm.
@@ -677,9 +702,8 @@ def flagMAD(data: DictOfSeries, field: ColumnName, flagger: Flagger, window: Fre
References
----------
[1] https://www.itl.nist.gov/div898/handbook/eda/section3/eda35h.htm
-
"""
- d = data[field].copy().mask(flagger.isFlagged(field))
+ d = data[field]
median = d.rolling(window=window, closed="both").median()
diff = (d - median).abs()
mad = diff.rolling(window=window, closed="both").median()
@@ -697,7 +721,7 @@ def flagMAD(data: DictOfSeries, field: ColumnName, flagger: Flagger, window: Fre
index = mask.index
mask.loc[index < index[0] + pd.to_timedelta(window)] = False
- flagger = flagger.setFlags(field, mask, **kwargs)
+ flagger[mask, field] = kwargs['flag']
return data, flagger
@@ -710,7 +734,7 @@ def flagOffset(
tolerance: float,
window: Union[IntegerWindow, FreqString],
rel_thresh: Optional[float]=None,
- numba_kickin: int=200000,
+ numba_kickin: int=200000, # todo: rm, not a user decision
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -780,13 +804,14 @@ def flagOffset(
if isinstance(window, int):
delta = getFreqDelta(dataseries.index)
+ window = delta * window
if not delta:
raise TypeError('Only offset string defined window sizes allowed for irrgegularily sampled timeseries')
- window = delta * window
# get all the entries preceding a significant jump
if thresh:
post_jumps = dataseries.diff().abs() > thresh
+
if rel_thresh:
s = np.sign(rel_thresh)
rel_jumps = s * (dataseries.shift(1).div(dataseries) - 1) > abs(rel_thresh)
@@ -798,34 +823,35 @@ def flagOffset(
post_jumps = post_jumps[post_jumps]
if post_jumps.empty:
return data, flagger
- # get all the entries preceeding a significant jump and its successors within "length" range
+
+ # get all the entries preceding a significant jump and its successors within "length" range
to_roll = post_jumps.reindex(dataseries.index, method="bfill", tolerance=window, fill_value=False).dropna()
- if not rel_thresh:
- # define spike testing function to roll with (no rel_check):
- def spikeTester(chunk, thresh=thresh, tol=tolerance):
- # signum change!!!
- chunk_stair = (np.sign(chunk[-2] - chunk[-1])*(chunk - chunk[-1]) < thresh)[::-1].cumsum()
+ if rel_thresh:
+
+ def spikeTester(chunk, thresh=abs(rel_thresh), tol=tolerance):
+ jump = chunk[-2] - chunk[-1]
+ thresh = thresh * abs(jump)
+ chunk_stair = (np.sign(jump) * (chunk - chunk[-1]) < thresh)[::-1].cumsum()
initial = np.searchsorted(chunk_stair, 2)
if initial == len(chunk):
return 0
if np.abs(chunk[- initial - 1] - chunk[-1]) < tol:
return initial - 1
- else:
- return 0
+ return 0
+
else:
- def spikeTester(chunk, thresh=abs(rel_thresh), tol=tolerance):
- jump = chunk[-2] - chunk[-1]
- thresh = thresh*abs(jump)
- chunk_stair = (np.sign(jump)*(chunk - chunk[-1]) < thresh)[::-1].cumsum()
+
+ # define spike testing function to roll with (no rel_check):
+ def spikeTester(chunk, thresh=thresh, tol=tolerance):
+ # signum change!!!
+ chunk_stair = (np.sign(chunk[-2] - chunk[-1]) * (chunk - chunk[-1]) < thresh)[::-1].cumsum()
initial = np.searchsorted(chunk_stair, 2)
if initial == len(chunk):
return 0
if np.abs(chunk[- initial - 1] - chunk[-1]) < tol:
return initial - 1
- else:
- return 0
-
+ return 0
to_roll = dataseries[to_roll]
roll_mask = pd.Series(False, index=to_roll.index)
@@ -835,6 +861,7 @@ def flagOffset(
engine = None if roll_mask.sum() < numba_kickin else 'numba'
result = roller.apply(spikeTester, raw=True, engine=engine)
result.index = map_i[result.index]
+
# correct the result: only those values define plateaus, that do not have
# values at their left starting point, that belong to other plateaus themself:
def calcResult(result):
@@ -850,7 +877,7 @@ def flagOffset(
cresult = calcResult(result)
cresult = cresult[cresult].index
- flagger = flagger.setFlags(field, cresult, **kwargs)
+ flagger[cresult, field] = kwargs['flag']
return data, flagger
@@ -914,9 +941,7 @@ def flagByGrubbs(
introduction to the grubbs test:
[1] https://en.wikipedia.org/wiki/Grubbs%27s_test_for_outliers
-
"""
-
data = data.copy()
datcol = data[field]
rate = getFreqDelta(datcol.index)
@@ -927,33 +952,38 @@ def flagByGrubbs(
to_group = pd.DataFrame(data={"ts": datcol.index, "data": datcol})
to_flag = pd.Series(False, index=datcol.index)
+
+ # period number defined test intervals
if isinstance(winsz, int):
- # period number defined test intervals
grouper_series = pd.Series(data=np.arange(0, datcol.shape[0]), index=datcol.index)
grouper_series_lagged = grouper_series + (winsz / 2)
grouper_series = grouper_series.transform(lambda x: x // winsz)
grouper_series_lagged = grouper_series_lagged.transform(lambda x: x // winsz)
partitions = to_group.groupby(grouper_series)
partitions_lagged = to_group.groupby(grouper_series_lagged)
+
+ # offset defined test intervals:
else:
- # offset defined test intervals:
partitions = to_group.groupby(pd.Grouper(freq=winsz))
+
for _, partition in partitions:
if partition.shape[0] > min_periods:
detected = smirnov_grubbs.two_sided_test_indices(partition["data"].values, alpha=alpha)
detected = partition["ts"].iloc[detected]
to_flag[detected.index] = True
- if check_lagged & isinstance(winsz, int):
+ if isinstance(winsz, int) and check_lagged:
to_flag_lagged = pd.Series(False, index=datcol.index)
+
for _, partition in partitions_lagged:
if partition.shape[0] > min_periods:
detected = smirnov_grubbs.two_sided_test_indices(partition["data"].values, alpha=alpha)
detected = partition["ts"].iloc[detected]
to_flag_lagged[detected.index] = True
- to_flag = to_flag & to_flag_lagged
- flagger = flagger.setFlags(field, loc=to_flag, **kwargs)
+ to_flag &= to_flag_lagged
+
+ flagger[to_flag, field] = kwargs['flag']
return data, flagger
@@ -994,7 +1024,7 @@ def flagRange(
# using .values is much faster
datacol = data[field].values
mask = (datacol < min) | (datacol > max)
- flagger[mask, field] = kwargs['flag'] # todo GL161
+ flagger[mask, field] = kwargs['flag']
return data, flagger
@@ -1057,22 +1087,28 @@ def flagCrossStatistic(
df = data[fields].loc[data[fields].index_of('shared')].to_df()
if isinstance(cross_stat, str):
+
if cross_stat == 'modZscore':
MAD_series = df.subtract(df.median(axis=1), axis=0).abs().median(axis=1)
- diff_scores = ((0.6745 * (df.subtract(df.median(axis=1), axis=0))).divide(MAD_series, axis=0)).abs()
+ diff_scores = (0.6745 * (df.subtract(df.median(axis=1), axis=0))).divide(MAD_series, axis=0).abs()
+
elif cross_stat == 'Zscore':
- diff_scores = (df.subtract(df.mean(axis=1), axis=0)).divide(df.std(axis=1), axis=0).abs()
+ diff_scores = df.subtract(df.mean(axis=1), axis=0).divide(df.std(axis=1), axis=0).abs()
+
else:
raise ValueError(cross_stat)
+
else:
+
try:
stat = getattr(df, cross_stat.__name__)(axis=1)
except AttributeError:
stat = df.aggregate(cross_stat, axis=1)
+
diff_scores = df.subtract(stat, axis=0).abs()
mask = diff_scores > thresh
for var in fields:
- flagger = flagger.setFlags(var, mask[var], **kwargs)
+ flagger[mask[var], field] = kwargs['flag']
return data, flagger
--
GitLab
From eab6883c17a00e7c5213a9a37c31b5f6fa0ff694 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 3 Mar 2021 01:26:23 +0100
Subject: [PATCH 023/180] fixed pattern.py
---
saqc/funcs/pattern.py | 12 +++---------
1 file changed, 3 insertions(+), 9 deletions(-)
diff --git a/saqc/funcs/pattern.py b/saqc/funcs/pattern.py
index 52437b89b..a33cdceae 100644
--- a/saqc/funcs/pattern.py
+++ b/saqc/funcs/pattern.py
@@ -1,17 +1,11 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from saqc.core.modules import base
from typing import Sequence, Union, Tuple, Optional
-from typing_extensions import Literal
-
import numpy as np
-
import dtw
import pywt
-
from mlxtend.evaluate import permutation_test
-
from dios.dios import DictOfSeries
from saqc.core.register import register
@@ -100,7 +94,7 @@ def flagPatternByDTW(
sz = len(ref)
mask = customRoller(dat, window=sz, min_periods=sz).apply(isPattern, raw=True)
- flagger = flagger.setFlags(field, loc=mask, **kwargs)
+ flagger[mask, field] = kwargs['flag']
return data, flagger
@@ -108,7 +102,7 @@ def flagPatternByDTW(
def flagPatternByWavelet(
data: DictOfSeries,
field: str,
- flagger: base,
+ flagger: Flagger,
ref_field: str,
max_distance: float=0.03,
normalize: bool=True,
@@ -172,5 +166,5 @@ def flagPatternByWavelet(
sz = len(ref)
mask = customRoller(dat, window=sz, min_periods=sz).apply(isPattern, raw=True)
- flagger = flagger.setFlags(field, loc=mask, **kwargs)
+ flagger[mask, field] = kwargs['flag']
return data, flagger
--
GitLab
From bfdb169c50a6164e3b2f3d21e0e30755dc8d6747 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 3 Mar 2021 01:53:30 +0100
Subject: [PATCH 024/180] fixed residues.py and scores.py
---
saqc/funcs/residues.py | 27 ++++++++++++++++++++-------
saqc/funcs/scores.py | 13 +++++++------
2 files changed, 27 insertions(+), 13 deletions(-)
diff --git a/saqc/funcs/residues.py b/saqc/funcs/residues.py
index 77cdcd948..16684a43f 100644
--- a/saqc/funcs/residues.py
+++ b/saqc/funcs/residues.py
@@ -99,10 +99,16 @@ def calculatePolynomialResidues(
Flags values may have changed relatively to the flagger input.
"""
- data, flagger = fitPolynomial(data, field, flagger, winsz, polydeg, numba=numba, eval_flags=eval_flags,
- min_periods=min_periods, return_residues=True, **kwargs)
-
- return data, flagger
+ return fitPolynomial(
+ data, field, flagger,
+ winsz=winsz,
+ polydeg=polydeg,
+ numba=numba,
+ eval_flags=eval_flags,
+ min_periods=min_periods,
+ return_residues=True,
+ **kwargs
+ )
@register(masking='field', module="residues")
@@ -118,7 +124,14 @@ def calculateRollingResidues(
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
- data, flagger = roll(data, field, flagger, winsz, func=func, eval_flags=eval_flags,
- min_periods=min_periods, center=center, return_residues=True, **kwargs)
+ return roll(
+ data, field, flagger,
+ winsz=winsz,
+ func=func,
+ eval_flags=eval_flags,
+ min_periods=min_periods,
+ center=center,
+ return_residues=True,
+ **kwargs
+ )
- return data, flagger
diff --git a/saqc/funcs/scores.py b/saqc/funcs/scores.py
index 8da17cbce..244f2a9e6 100644
--- a/saqc/funcs/scores.py
+++ b/saqc/funcs/scores.py
@@ -108,17 +108,18 @@ def assignKNNScore(
References
----------
-
[1] https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.NearestNeighbors.html
-
"""
data = data.copy()
fields = toSequence(fields)
+
val_frame = data[fields]
score_index = val_frame.index_of("shared")
score_ser = pd.Series(np.nan, index=score_index, name=target_field)
+
val_frame = val_frame.loc[val_frame.index_of("shared")].to_df()
val_frame.dropna(inplace=True)
+
if not trafo_on_partition:
val_frame = val_frame.transform(trafo)
@@ -154,11 +155,11 @@ def assignKNNScore(
score_ser[partition.index] = resids
- score_flagger = flagger.initFlags(score_ser)
-
+ # this unconditionally overwrite a column,
+ # may we should fire a warning ? -- palmb
if target_field in flagger.columns:
- flagger = flagger.slice(drop=target_field)
+ flagger.drop(target_field)
+ flagger[target_field] = score_ser
- flagger = flagger.merge(score_flagger)
data[target_field] = score_ser
return data, flagger
--
GitLab
From 4d5e76eeafd7cfaab21c186351bbaa0422bd158f Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 3 Mar 2021 02:16:43 +0100
Subject: [PATCH 025/180] fixed tools.py
---
saqc/funcs/tools.py | 45 ++++++++++++++++-----------------------------
1 file changed, 16 insertions(+), 29 deletions(-)
diff --git a/saqc/funcs/tools.py b/saqc/funcs/tools.py
index 9ad92f94b..3e49f762b 100644
--- a/saqc/funcs/tools.py
+++ b/saqc/funcs/tools.py
@@ -8,6 +8,7 @@ import numpy as np
from dios import DictOfSeries
+from saqc.common import *
from saqc.core.register import register
from saqc.flagger import Flagger
from saqc.lib.tools import periodicMask
@@ -39,15 +40,12 @@ def copy(data: DictOfSeries, field: str, flagger: Flagger, new_field: str, **kwa
The flagger object, holding flags and additional Informations related to `data`.
Flags shape may have changed relatively to the flagger input.
"""
-
if new_field in flagger.columns.union(data.columns):
raise ValueError(f"{field}: field already exist")
- flags, extras = flagger.getFlags(field, full=True)
- newflagger = flagger.replaceField(new_field, flags=flags, **extras)
- newdata = data.copy()
- newdata[new_field] = data[field].copy()
- return newdata, newflagger
+ data[new_field] = data[field].copy()
+ flagger.history[new_field] = flagger.history[field]
+ return data, flagger
@register(masking='none', module="tools")
@@ -73,10 +71,8 @@ def drop(data: DictOfSeries, field: str, flagger: Flagger, **kwargs) -> Tuple[Di
The flagger object, holding flags and additional Informations related to `data`.
Flags shape may have changed relatively to the flagger input.
"""
-
- data = data.copy()
del data[field]
- flagger = flagger.replaceField(field, flags=None)
+ del flagger[field]
return data, flagger
@@ -103,19 +99,10 @@ def rename(data: DictOfSeries, field: str, flagger: Flagger, new_name: str, **kw
flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
"""
- # store
- s = data[field]
- f, e = flagger.getFlags(field, full=True)
-
- # delete
- data = data.copy()
+ data[new_name] = data[field]
+ flagger.history[new_name] = flagger.history[field]
del data[field]
- flagger = flagger.replaceField(field, flags=None)
-
- # insert
- data[new_name] = s
- flagger = flagger.replaceField(new_name, inplace=True, flags=f, **e)
-
+ del flagger[field]
return data, flagger
@@ -128,24 +115,25 @@ def mask(
mask_var: Optional[str]=None,
period_start: Optional[str]=None,
period_end: Optional[str]=None,
- include_bounds: bool=True
+ include_bounds: bool=True,
+ **kwargs,
) -> Tuple[DictOfSeries, Flagger]:
"""
This function realizes masking within saqc.
Due to some inner saqc mechanics, it is not straight forwardly possible to exclude
- values or datachunks from flagging routines. This function replaces flags with np.nan
+ values or datachunks from flagging routines. This function replaces flags with UNFLAGGED
value, wherever values are to get masked. Furthermore, the masked values get replaced by
np.nan, so that they dont effect calculations.
Here comes a recipe on how to apply a flagging function only on a masked chunk of the variable field:
- 1. dublicate "field" in the input data (proc_copy)
- 2. mask the dublicated data (modelling_mask)
+ 1. dublicate "field" in the input data (copy)
+ 2. mask the dublicated data (mask)
3. apply the tests you only want to be applied onto the masked data chunks (saqc_tests)
4. project the flags, calculated on the dublicated and masked data onto the original field data
- (proc_projectFlags or flagGeneric)
- 5. drop the dublicated data (proc_drop)
+ (projectFlags or flagGeneric)
+ 5. drop the dublicated data (drop)
To see an implemented example, checkout flagSeasonalRange in the saqc.functions module
@@ -239,6 +227,5 @@ def mask(
raise ValueError("Keyword passed as masking mode is unknown ({})!".format(mode))
data.aloc[to_mask, field] = np.nan
- flagger = flagger.setFlags(field, loc=to_mask, flag=np.nan, force=True)
-
+ flagger[to_mask, field] = UNFLAGGED
return data, flagger
--
GitLab
From 4e51d3f6a06b89c953409f9a5b79581a36c9840f Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 3 Mar 2021 13:09:33 +0100
Subject: [PATCH 026/180] fixed refactor bug in scores.py
---
saqc/funcs/scores.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/saqc/funcs/scores.py b/saqc/funcs/scores.py
index 244f2a9e6..1c670b287 100644
--- a/saqc/funcs/scores.py
+++ b/saqc/funcs/scores.py
@@ -8,6 +8,7 @@ import pandas as pd
from dios import DictOfSeries
+from saqc.common import *
from saqc.core.register import register
from saqc.flagger import Flagger
from saqc.lib import ts_operators as ts_ops
@@ -155,11 +156,10 @@ def assignKNNScore(
score_ser[partition.index] = resids
- # this unconditionally overwrite a column,
- # may we should fire a warning ? -- palmb
+ # todo: this unconditionally overwrite a column, may we should fire a warning ? -- palmb
if target_field in flagger.columns:
flagger.drop(target_field)
- flagger[target_field] = score_ser
+ flagger[target_field] = pd.Series(UNFLAGGED, index=score_ser.index, dtype=float)
data[target_field] = score_ser
return data, flagger
--
GitLab
From 441acd0d46e51c194331314722776d60bfc68557 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 3 Mar 2021 15:32:58 +0100
Subject: [PATCH 027/180] added reindex to History, and added tests for that;
fixed old bug in test History
---
saqc/flagger/history.py | 34 +++++++++++++++++++++++++------
test/flagger/test_history.py | 39 ++++++++++++++++++++++++++++++++++++
2 files changed, 67 insertions(+), 6 deletions(-)
diff --git a/saqc/flagger/history.py b/saqc/flagger/history.py
index 9eeb9aa9f..65759b25c 100644
--- a/saqc/flagger/history.py
+++ b/saqc/flagger/history.py
@@ -4,6 +4,7 @@ from __future__ import annotations
from typing import Tuple, Type
import pandas as pd
import numpy as np
+from saqc.common import *
class History:
@@ -149,7 +150,9 @@ class History:
-------
History
"""
- # internal detail:
+ # Note:
+ # all following code must handle a passed empty series
+
# ensure continuous increasing columns
assert 0 <= pos <= len(self)
@@ -191,11 +194,8 @@ class History:
"""
s = self._validate_value(value)
- if s.empty:
- raise ValueError('Cannot append empty pd.Series')
-
- if not self.empty and not s.index.equals(self.index):
- raise ValueError("Index must be equal to FH's index")
+ if len(self) > 0 and not s.index.equals(self.index):
+ raise ValueError("Index must be equal to FlagHistory index")
self._insert(value, pos=len(self), force=force)
return self
@@ -279,6 +279,28 @@ class History:
"""
return self._constructor(hist=self, copy=deep)
+ def reindex(self, index: pd.Index, fill_value_last: float = UNFLAGGED) -> History:
+ """
+ Reindex the History. Be careful this alters the past.
+
+ Parameters
+ ----------
+ index : pd.Index
+ the index to reindex to.
+ fill_value_last : float, default 0
+ value to fill nan's (UNTOUCHED) in the last column. Defaults to 0 (UNFLAGGED).
+
+ Returns
+ -------
+ History
+ """
+ self.hist = self.hist.reindex(index=index, copy=False, fill_value=np.nan)
+ self.mask = self.mask.reindex(index=index, copy=False, fill_value=False)
+ # Note: all following code must handle empty frames
+ self.hist.iloc[:, -1:] = self.hist.iloc[:, -1:].fillna(fill_value_last)
+ self.mask.iloc[:, -1:] = True
+ return self
+
def __copy__(self, deep: bool = True):
return self.copy(deep=deep)
diff --git a/test/flagger/test_history.py b/test/flagger/test_history.py
index e1957615d..d6a084827 100644
--- a/test/flagger/test_history.py
+++ b/test/flagger/test_history.py
@@ -102,7 +102,11 @@ def check_invariants(hist):
# or the entire row is True
if not hist.empty:
idxmax = hist.mask.idxmax(axis=1)
+ print(f'idxmax: {idxmax}')
for row, col in idxmax.items():
+ # this is contra intuitive, it gets the positional (for iloc)
+ row = idxmax.index.get_loc(row)
+
assert all(hist.mask.iloc[row, :col] == False)
assert all(hist.mask.iloc[row, col:] == True)
@@ -183,6 +187,41 @@ def test_copy(data):
assert shallow.mask is hist.mask
+@pytest.mark.parametrize('data', data + [None])
+def test_reindex_trivial_cases(data):
+ df = pd.DataFrame(data, dtype=float)
+ orig = History(hist=df)
+
+ # checks
+ for index in [df.index, pd.Index([])]:
+ hist = orig.copy()
+ ref = hist.reindex(index)
+ assert ref is hist # check if working inplace
+ check_invariants(hist)
+
+
+@pytest.mark.parametrize('data', data + [None])
+def test_reindex_missing_indicees(data):
+ df = pd.DataFrame(data, dtype=float)
+ hist = History(hist=df)
+ index = df.index[1:-1]
+ # checks
+ ref = hist.reindex(index)
+ assert ref is hist # check if working inplace
+ check_invariants(hist)
+
+
+@pytest.mark.parametrize('data', data + [None])
+def test_reindex_extra_indicees(data):
+ df = pd.DataFrame(data, dtype=float)
+ hist = History(hist=df)
+ index = df.index.append(pd.Index(range(len(df.index), len(df.index) + 5)))
+ # checks
+ ref = hist.reindex(index)
+ assert ref is hist # check if working inplace
+ check_invariants(hist)
+
+
@pytest.fixture(scope='module')
def __hist():
# this FH is filled by
--
GitLab
From 6a684dbbe40b137a642a9b40831969c620609564 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 3 Mar 2021 18:18:23 +0100
Subject: [PATCH 028/180] fixed resampling.py, added History changing function
to flags.py
---
saqc/flagger/flags.py | 57 +++++++++++++++++-
saqc/funcs/resampling.py | 122 ++++++++++++++++++++++++---------------
2 files changed, 131 insertions(+), 48 deletions(-)
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index 175833ff5..537cf9178 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -275,7 +275,11 @@ class Flags:
return str(self.toDios()).replace('DictOfSeries', type(self).__name__)
-def initFlagsLike(reference: Union[pd.Series, DictLike, Flags], initial_value: float = UNFLAGGED) -> Flags:
+def initFlagsLike(
+ reference: Union[pd.Series, DictLike, Flags],
+ initial_value: float = UNFLAGGED,
+ name: str = None,
+) -> Flags:
"""
Create empty Flags, from an reference data structure.
@@ -287,6 +291,12 @@ def initFlagsLike(reference: Union[pd.Series, DictLike, Flags], initial_value: f
initial_value : float, default 0
value to initialize the columns with
+ name : str, default None
+ Only respected if `reference` is of type ``pd.Series``.
+ The column name that is used for the Flags. If ``None``
+ the name of the series itself is taken, if this is also
+ `None`, a ValueError is raised.
+
Notes
-----
Implementation detail:
@@ -307,7 +317,13 @@ def initFlagsLike(reference: Union[pd.Series, DictLike, Flags], initial_value: f
reference = reference._data
if isinstance(reference, pd.Series):
- reference = reference.to_frame('f0')
+ if name is None:
+ name = reference.name
+ if name is None:
+ raise ValueError("Either the passed series must be named or a name must be passed")
+ if not isinstance(name, str):
+ raise TypeError(f"name must be str not '{type(name).__name__}'")
+ reference = reference.to_frame(name=name)
for k, item in reference.items():
@@ -327,5 +343,42 @@ def initFlagsLike(reference: Union[pd.Series, DictLike, Flags], initial_value: f
return Flags(result)
+def applyFunctionOnHistory(flags: Flags, column, hist_func, hist_kws, mask_func, mask_kws, last_column=None):
+ """
+ Apply function on history.
+
+ Two functions must be given. Both are called for each column in the History. One on History.hist, the
+ other on History.mask. Both take a pd.Series as first arg, which is the column from hist or mask respectively.
+
+ Parameters
+ ----------
+ flags :
+ column :
+ hist_func :
+ hist_kws :
+ mask_func :
+ mask_kws :
+ last_column :
+
+ Returns
+ -------
+
+ """
+ flags = flags.copy()
+ history = flags.history[column]
+ new_history = History()
+ for pos in history.columns:
+ new_history.hist[pos] = hist_func(history.hist[pos], **hist_kws)
+ new_history.mask[pos] = mask_func(history.mask[pos], **mask_kws)
+
+ if last_column is None:
+ new_history.mask.iloc[:, -1:] = True
+ else:
+ new_history.append(last_column, force=True)
+
+ flags.history[column] = new_history
+ return flags
+
+
# for now we keep this name
Flagger = Flags
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index bf809340e..27d44b30d 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -13,11 +13,12 @@ from dios import DictOfSeries
from saqc.common import *
from saqc.core.register import register
-from saqc.flagger import Flagger
+from saqc.flagger import Flagger, initFlagsLike, History
from saqc.funcs.tools import copy, drop, rename
from saqc.funcs.interpolation import interpolateIndex
from saqc.lib.tools import dropper, evalFreqStr
from saqc.lib.ts_operators import shift2Freq, aggregate2Freq
+from saqc.flagger.flags import applyFunctionOnHistory
logger = logging.getLogger("SaQC")
@@ -42,7 +43,7 @@ def aggregate(
value_func,
flag_func: Callable[[pd.Series], float]=np.nanmax,
method: Literal["fagg", "bagg", "nagg"]="nagg",
- to_drop: Optional[Union[Any, Sequence[Any]]]=None,
+ to_drop: Optional[Union[Any, Sequence[Any]]]=None, # todo: rm, use to_mask instead
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -343,7 +344,7 @@ def shift(
method: Literal["fshift", "bshift", "nshift"]="nshift",
to_drop: Optional[Union[Any, Sequence[Any]]]=None,
empty_intervals_flag: Optional[str]=None,
- freq_check: Optional[Literal["check", "auto"]]=None,
+ freq_check: Optional[Literal["check", "auto"]]=None, # todo: rm, not a user decision
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
@@ -417,7 +418,7 @@ def _shift(
"""
data = data.copy()
datcol = data[field]
- flagscol = flagger.getFlags(field)
+ flagscol = flagger[field]
if empty_intervals_flag is None:
empty_intervals_flag = UNFLAGGED
@@ -426,20 +427,33 @@ def _shift(
drop_mask |= datcol.isna()
datcol[drop_mask] = np.nan
datcol.dropna(inplace=True)
- freq = evalFreqStr(freq, freq_check, datcol.index)
+ flagscol.drop(drop_mask[drop_mask].index, inplace=True)
+
+ # create a dummys
if datcol.empty:
- data[field] = datcol
- reshaped_flagger = flagger.initFlags(datcol).setFlags(field, flag=flagscol, force=True, inplace=True, **kwargs)
- flagger = flagger.slice(drop=field).merge(reshaped_flagger, subset=[field], inplace=True)
- return data, flagger
+ datcol = pd.Series([], index=pd.DatetimeIndex([]), name=field)
+ flagscol = pd.Series([], index=pd.DatetimeIndex([]), name=field)
- flagscol.drop(drop_mask[drop_mask].index, inplace=True)
+ # clear the past
+ flagger.history[field] = flagger.history[field].reindex(datcol.index)
+ flagger[field] = flagscol
+
+ # do the shift, we need to process the history manually
+ else:
+ freq = evalFreqStr(freq, freq_check, datcol.index)
+ datcol = shift2Freq(datcol, method, freq, fill_value=np.nan)
+
+ # after next 3 lines we leave history in unstable state
+ # but the following append will fix this
+ history = flagger.history[field]
+ history.hist = shift2Freq(history.hist, method, freq, fill_value=UNTOUCHED)
+ history.mask = shift2Freq(history.mask, method, freq, fill_value=False)
+
+ flagscol = shift2Freq(flagscol, method, freq, fill_value=empty_intervals_flag)
+ history.append(flagscol, force=True)
+ flagger.history[field] = history
- datcol = shift2Freq(datcol, method, freq, fill_value=np.nan)
- flagscol = shift2Freq(flagscol, method, freq, fill_value=empty_intervals_flag)
data[field] = datcol
- reshaped_flagger = flagger.initFlags(datcol).setFlags(field, flag=flagscol, force=True, inplace=True, **kwargs)
- flagger = flagger.slice(drop=field).merge(reshaped_flagger, subset=[field], inplace=True)
return data, flagger
@@ -546,7 +560,7 @@ def resample(
data = data.copy()
datcol = data[field]
- flagscol = flagger.getFlags(field)
+ flagscol = flagger[field]
if empty_intervals_flag is None:
empty_intervals_flag = BAD
@@ -554,41 +568,56 @@ def resample(
datcol.drop(datcol[drop_mask].index, inplace=True)
freq = evalFreqStr(freq, freq_check, datcol.index)
flagscol.drop(flagscol[drop_mask].index, inplace=True)
- if all_na_2_empty:
- if datcol.dropna().empty:
- datcol = pd.Series([], index=pd.DatetimeIndex([]), name=field)
- if datcol.empty:
- # for consistency reasons - return empty data/flags column when there is no valid data left
- # after filtering.
- data[field] = datcol
- reshaped_flagger = flagger.initFlags(datcol).setFlags(field, flag=flagscol, force=True, inplace=True, **kwargs)
- flagger = flagger.slice(drop=field).merge(reshaped_flagger, subset=[field], inplace=True)
- return data, flagger
+ # create a dummys
+ if all_na_2_empty and datcol.dropna().empty:
+
+ datcol = pd.Series([], index=pd.DatetimeIndex([]), name=field)
+ flagscol = pd.Series([], index=pd.DatetimeIndex([]), name=field)
+
+ # clear the past
+ flagger.history[field] = flagger.history[field].reindex(datcol.index)
+ flagger[field] = flagscol
+
+ # do the resampling
+ else:
+ datcol = aggregate2Freq(
+ datcol,
+ method,
+ freq,
+ agg_func,
+ fill_value=np.nan,
+ max_invalid_total=max_invalid_total_d,
+ max_invalid_consec=max_invalid_consec_d,
+ )
- datcol = aggregate2Freq(
- datcol,
- method,
- freq,
- agg_func,
- fill_value=np.nan,
- max_invalid_total=max_invalid_total_d,
- max_invalid_consec=max_invalid_consec_d,
- )
- flagscol = aggregate2Freq(
- flagscol,
- method,
- freq,
- flag_agg_func,
- fill_value=empty_intervals_flag,
- max_invalid_total=max_invalid_total_f,
- max_invalid_consec=max_invalid_consec_f,
- )
+ flagscol = aggregate2Freq(
+ flagscol,
+ method,
+ freq,
+ flag_agg_func,
+ fill_value=empty_intervals_flag,
+ max_invalid_total=max_invalid_total_f,
+ max_invalid_consec=max_invalid_consec_f,
+ )
+
+ kws = dict(
+ method=method,
+ freq=freq,
+ agg_func=flag_agg_func,
+ fill_value=UNTOUCHED,
+ max_invalid_total=max_invalid_total_f,
+ max_invalid_consec=max_invalid_consec_f,
+ )
+
+ flagger = applyFunctionOnHistory(
+ flagger, field,
+ hist_func=aggregate2Freq, hist_kws=kws,
+ mask_func=aggregate2Freq, mask_kws=kws,
+ last_column=flagscol
+ )
- # data/flags reshaping:
data[field] = datcol
- reshaped_flagger = flagger.initFlags(datcol).setFlags(field, flag=flagscol, force=True, inplace=True, **kwargs)
- flagger = flagger.slice(drop=field).merge(reshaped_flagger, subset=[field], inplace=True)
return data, flagger
@@ -671,6 +700,7 @@ def reindexFlags(
"""
# TODO: This needs a refactoring
+ raise NotImplementedError("currently not available - rewrite needed")
flagscol, metacols = flagger.getFlags(source, full=True)
if flagscol.empty:
--
GitLab
From 527f72465025cce7b10443e317ae00b7fec4bce5 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 3 Mar 2021 18:22:52 +0100
Subject: [PATCH 029/180] refactored dropper -> getDropMask; minor fixes;
added/rewrite TODOs
---
saqc/core/core.py | 2 +-
saqc/funcs/changepoints.py | 4 ++--
saqc/funcs/interpolation.py | 4 ++--
saqc/funcs/resampling.py | 12 ++++++------
saqc/lib/tools.py | 8 +++++---
saqc/lib/ts_operators.py | 8 ++++----
6 files changed, 20 insertions(+), 18 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index e850385ae..de726326d 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -49,7 +49,7 @@ def _handleErrors(exc: Exception, field: str, control: APIController, func: SaQC
raise exc
-# todo: shouldt this go to Saqc.__init__ ?
+# todo: shouldt the code/function go to Saqc.__init__ ?
def _prepInput(data, flags):
dios_like = (dios.DictOfSeries, pd.DataFrame)
diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py
index 9657b53d0..f11b413ba 100644
--- a/saqc/funcs/changepoints.py
+++ b/saqc/funcs/changepoints.py
@@ -30,7 +30,7 @@ def flagChangePoints(
fwd_window: Optional[FreqString]=None,
min_periods_fwd: Optional[IntegerWindow]=None,
closed: Literal["right", "left", "both", "neither"]="both",
- try_to_jit: bool=True, # todo rm
+ try_to_jit: bool=True, # todo rm, not a user decision
reduce_window: FreqString=None,
reduce_func: Callable[[np.ndarray, np.ndarray], int]=lambda x, _: x.argmax(),
**kwargs
@@ -107,7 +107,7 @@ def assignChangePointCluster(
fwd_window: str=None,
min_periods_fwd: Optional[int]=None,
closed: Literal["right", "left", "both", "neither"]="both",
- try_to_jit: bool=True, # todo: rm
+ try_to_jit: bool=True, # todo: rm, not a user decision
reduce_window: str=None,
reduce_func: Callable[[np.ndarray, np.ndarray], float]=lambda x, _: x.argmax(),
model_by_resids: bool=False,
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index 98508c88b..3a5d73cb2 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -13,7 +13,7 @@ from saqc.common import *
from saqc.core.register import register
from saqc.flagger import Flagger
-from saqc.lib.tools import toSequence, evalFreqStr, dropper
+from saqc.lib.tools import toSequence, evalFreqStr, getDropMask
from saqc.lib.ts_operators import interpolateNANs
@@ -259,7 +259,7 @@ def interpolateIndex(
if empty_intervals_flag is None:
empty_intervals_flag = BAD
- drop_mask = dropper(field, to_drop, flagger, BAD)
+ drop_mask = getDropMask(field, to_drop, flagger, BAD)
drop_mask |= flagscol.isna()
drop_mask |= datcol.isna()
datcol[drop_mask] = np.nan
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 27d44b30d..d6fed78ee 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -16,7 +16,7 @@ from saqc.core.register import register
from saqc.flagger import Flagger, initFlagsLike, History
from saqc.funcs.tools import copy, drop, rename
from saqc.funcs.interpolation import interpolateIndex
-from saqc.lib.tools import dropper, evalFreqStr
+from saqc.lib.tools import getDropMask, evalFreqStr
from saqc.lib.ts_operators import shift2Freq, aggregate2Freq
from saqc.flagger.flags import applyFunctionOnHistory
@@ -43,7 +43,7 @@ def aggregate(
value_func,
flag_func: Callable[[pd.Series], float]=np.nanmax,
method: Literal["fagg", "bagg", "nagg"]="nagg",
- to_drop: Optional[Union[Any, Sequence[Any]]]=None, # todo: rm, use to_mask instead
+ to_drop: Optional[Union[Any, Sequence[Any]]]=None,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -344,7 +344,7 @@ def shift(
method: Literal["fshift", "bshift", "nshift"]="nshift",
to_drop: Optional[Union[Any, Sequence[Any]]]=None,
empty_intervals_flag: Optional[str]=None,
- freq_check: Optional[Literal["check", "auto"]]=None, # todo: rm, not a user decision
+ freq_check: Optional[Literal["check", "auto"]]=None, # todo: not a user decision
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
@@ -423,7 +423,7 @@ def _shift(
if empty_intervals_flag is None:
empty_intervals_flag = UNFLAGGED
- drop_mask = dropper(field, to_drop, flagger, BAD)
+ drop_mask = getDropMask(field, to_drop, flagger, BAD)
drop_mask |= datcol.isna()
datcol[drop_mask] = np.nan
datcol.dropna(inplace=True)
@@ -564,7 +564,7 @@ def resample(
if empty_intervals_flag is None:
empty_intervals_flag = BAD
- drop_mask = dropper(field, to_drop, flagger, [])
+ drop_mask = getDropMask(field, to_drop, flagger, [])
datcol.drop(datcol[drop_mask].index, inplace=True)
freq = evalFreqStr(freq, freq_check, datcol.index)
flagscol.drop(flagscol[drop_mask].index, inplace=True)
@@ -751,7 +751,7 @@ def reindexFlags(
#
# starting with the dropping and its memorization:
- drop_mask = dropper(field, to_drop, flagger, BAD)
+ drop_mask = getDropMask(field, to_drop, flagger, BAD)
drop_mask |= target_datcol.isna()
target_flagscol_drops = target_flagscol[drop_mask]
target_flagscol.drop(drop_mask[drop_mask].index, inplace=True)
diff --git a/saqc/lib/tools.py b/saqc/lib/tools.py
index 01d534367..b70f660dd 100644
--- a/saqc/lib/tools.py
+++ b/saqc/lib/tools.py
@@ -308,13 +308,15 @@ def isQuoted(string):
return bool(re.search(r"'.*'|\".*\"", string))
-def dropper(field, to_drop, flagger, default):
- drop_mask = pd.Series(False, flagger.getFlags(field).index)
+# todo: GL167
+def getDropMask(field, to_drop, flagger, default):
+ drop_mask = pd.Series(False, index=flagger[field].index)
if to_drop is None:
to_drop = default
to_drop = toSequence(to_drop)
if len(to_drop) > 0:
- drop_mask |= flagger.isFlagged(field, flag=to_drop)
+ # drop_mask |= flagger.isFlagged(field, flag=to_drop)
+ drop_mask |= flagger[field] == to_drop
return drop_mask
diff --git a/saqc/lib/ts_operators.py b/saqc/lib/ts_operators.py
index bd698fc00..8f1f86cef 100644
--- a/saqc/lib/ts_operators.py
+++ b/saqc/lib/ts_operators.py
@@ -279,8 +279,8 @@ def aggregate2Freq(
}
# filter data for invalid patterns (since filtering is expensive we pre-check if it is demanded)
- if (max_invalid_total is not None) | (max_invalid_consec is not None):
- if pd.isnull(fill_value):
+ if max_invalid_total is not None or max_invalid_consec is not None:
+ if pd.isna(fill_value):
temp_mask = data.isna()
else:
temp_mask = data == fill_value
@@ -313,8 +313,8 @@ def aggregate2Freq(
except AttributeError:
data = data_resampler.apply(agg_func)
- # since loffset keyword of pandas.resample "discharges" after one use of the resampler (pandas logic) - we correct the
- # resampled labels offset manually, if necessary.
+ # since loffset keyword of pandas.resample "discharges" after one use of the resampler (pandas logic),
+ # we correct the resampled labels offset manually, if necessary.
if method == "nagg":
data.index = data.index.shift(freq=pd.Timedelta(freq) / 2)
empty_intervals.index = empty_intervals.index.shift(freq=pd.Timedelta(freq) / 2)
--
GitLab
From 3e1fcfb297bc9c554d7a1a1861532de8d75ddc60 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 3 Mar 2021 23:57:17 +0100
Subject: [PATCH 030/180] fixed generic.py
---
saqc/common.py | 6 +++---
saqc/funcs/generic.py | 45 +++++++++++++++++++++++--------------------
2 files changed, 27 insertions(+), 24 deletions(-)
diff --git a/saqc/common.py b/saqc/common.py
index 9d4e3d504..013797048 100644
--- a/saqc/common.py
+++ b/saqc/common.py
@@ -12,9 +12,9 @@ __all__ = [
import numpy as np
UNTOUCHED = np.nan
-UNFLAGGED = 0
-DOUBTFUL = 25
-BAD = 255
+UNFLAGGED = 0.
+DOUBTFUL = 25.
+BAD = 255.
# aliases
GOOD = UNFLAGGED
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index 6bb3fdeb3..85f6e1300 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -13,17 +13,25 @@ from dios import DictOfSeries
from saqc.common import *
from saqc.core.register import register
from saqc.core.visitor import ENVIRONMENT
-from saqc.flagger import Flagger
+from saqc.flagger import Flagger, initFlagsLike
+import operator as op
-def _dslIsFlagged(flagger: Flagger, var: pd.Series, flag: Any=None, comparator: str=">=") -> Union[pd.Series, DictOfSeries]:
+_OP = {'<': op.lt, '<=': op.le, '==': op.eq, '!=': op.ne, '>': op.gt, '>=': op.ge}
+
+
+def _dslIsFlagged(
+ flagger: Flagger, var: pd.Series, flag: Any = None, comparator: str = ">="
+) -> Union[pd.Series, DictOfSeries]:
"""
helper function for `flag`
"""
- return flagger.isFlagged(var.name, flag=flag, comparator=comparator)
+ comparison = _OP[comparator]
+ return comparison(flagger[var.name], flag)
-def _execGeneric(flagger: Flagger, data: DictOfSeries, func: Callable[[pd.Series], pd.Series], field: str, nodata: float) -> pd.Series:
+def _execGeneric(flagger: Flagger, data: DictOfSeries, func: Callable[[pd.Series], pd.Series], field: str,
+ nodata: float) -> pd.Series:
# TODO:
# - check series.index compatibility
# - field is only needed to translate 'this' parameters
@@ -53,7 +61,8 @@ def _execGeneric(flagger: Flagger, data: DictOfSeries, func: Callable[[pd.Series
@register(masking='all', module="generic")
-def process(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd.Series], pd.Series], nodata: float=np.nan, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+def process(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd.Series], pd.Series],
+ nodata: float = np.nan, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
generate/process data with generically defined functions.
@@ -105,24 +114,18 @@ def process(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd
"""
data[field] = _execGeneric(flagger, data, func, field, nodata).squeeze()
- # NOTE:
- # The flags to `field` will be (re-)set to UNFLAGGED
- # That leads to the following problem:
- # flagger.merge merges the given flaggers, if
- # `field` did already exist before the call to `procGeneric`
- # but with a differing index, we end up with:
- # len(data[field]) != len(flagger.getFlags(field))
- # see: test/funcs/test_generic_functions.py::test_procGenericMultiple
- # TODO:
- # We need a way to simply overwrite a given flagger column, maybe
- # an optional keyword to merge ?
- flagger = flagger.merge(flagger.initFlags(data[field]))
+ # todo: the former comment wished to overwrite the column, but i'm not sure -- palmb
+ if field in flagger:
+ flagger.drop(field)
+
+ flagger[field] = initFlagsLike(data[field])[field]
return data, flagger
@register(masking='all', module="generic")
-def flag(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd.Series], pd.Series], nodata: float=np.nan, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+def flag(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd.Series], pd.Series],
+ nodata: float = np.nan, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
a function to flag a data column by evaluation of a generic expression.
@@ -206,12 +209,12 @@ def flag(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd.Se
if not np.issubdtype(mask.dtype, np.bool_):
raise TypeError(f"generic expression does not return a boolean array")
- if field not in flagger.getFlags():
- flagger = flagger.merge(flagger.initFlags(data=pd.Series(index=mask.index, name=field)))
+ if field not in flagger.columns:
+ flagger[field] = pd.Series(UNFLAGGED, index=mask.index, name=field)
# if flagger.getFlags(field).empty:
# flagger = flagger.merge(
# flagger.initFlags(
# data=pd.Series(name=field, index=mask.index, dtype=np.float64)))
- flagger = flagger.setFlags(field=field, loc=mask, **kwargs)
+ flagger[mask, field] = kwargs['flag']
return data, flagger
--
GitLab
From 4c90a78dc6ef52bf2974a82dc067cfcafb047919 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Thu, 4 Mar 2021 02:08:37 +0100
Subject: [PATCH 031/180] minor bugfixes
---
saqc/core/reader.py | 1 -
saqc/funcs/flagtools.py | 3 ++-
saqc/funcs/generic.py | 4 ++--
saqc/funcs/outliers.py | 4 ++--
4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/saqc/core/reader.py b/saqc/core/reader.py
index 9fe240d2b..32d5b6985 100644
--- a/saqc/core/reader.py
+++ b/saqc/core/reader.py
@@ -80,7 +80,6 @@ def _parseConfig(df, flagger):
)
control = ConfigController(
- masking=func.masking,
plot=plot,
lineno=lineno + 2,
expression=expr
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index c72868927..99337a70b 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -67,7 +67,8 @@ def clearFlags(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs
Returns
-------
- data, flagger: DictOfSeries, Flagger
+ data : DictOfSeries
+ flagger : Flagger
See Also
--------
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index 85f6e1300..8753ca154 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -125,7 +125,7 @@ def process(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd
@register(masking='all', module="generic")
def flag(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd.Series], pd.Series],
- nodata: float = np.nan, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+ nodata: float = np.nan, flag=BAD, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
a function to flag a data column by evaluation of a generic expression.
@@ -216,5 +216,5 @@ def flag(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd.Se
# flagger = flagger.merge(
# flagger.initFlags(
# data=pd.Series(name=field, index=mask.index, dtype=np.float64)))
- flagger[mask, field] = kwargs['flag']
+ flagger[mask, field] = flag
return data, flagger
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index 22bace68c..bc7f5e06c 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -80,7 +80,7 @@ def flagByStray(
[1] Talagala, P. D., Hyndman, R. J., & Smith-Miles, K. (2019). Anomaly detection in high dimensional data.
arXiv preprint arXiv:1908.04000.
"""
- scores = data[field].dropna(inplace=True)
+ scores = data[field].dropna()
if scores.empty:
return data, flagger
@@ -1109,6 +1109,6 @@ def flagCrossStatistic(
mask = diff_scores > thresh
for var in fields:
- flagger[mask[var], field] = kwargs['flag']
+ flagger[mask[var], var] = kwargs['flag']
return data, flagger
--
GitLab
From d2785dae7567d0616efbf9022bb69fa4daa8705b Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Thu, 4 Mar 2021 02:11:42 +0100
Subject: [PATCH 032/180] fixed flagger syntax in most tests
---
test/common.py | 23 +--
test/core/test_core.py | 52 +++---
.../{test_core_new.py => test_creation.py} | 0
test/core/test_masking.py | 3 +-
test/core/test_reader.py | 15 +-
test/flagger/test_dmpflagger.py | 4 +-
test/flagger/test_positionalflagger.py | 3 +-
test/funcs/conftest.py | 92 +++++-----
test/funcs/test_constants_detection.py | 24 ++-
test/funcs/test_functions.py | 165 ++++++++++--------
test/funcs/test_generic_api_functions.py | 46 ++---
test/funcs/test_generic_config_functions.py | 69 ++++----
test/funcs/test_pattern_rec.py | 34 ++--
test/funcs/test_spikes_detection.py | 61 +++----
14 files changed, 288 insertions(+), 303 deletions(-)
rename test/core/{test_core_new.py => test_creation.py} (100%)
diff --git a/test/common.py b/test/common.py
index e07cc5cfb..f774cd5ed 100644
--- a/test/common.py
+++ b/test/common.py
@@ -24,29 +24,22 @@ from hypothesis.strategies._internal.types import _global_type_lookup
from dios import DictOfSeries
+from saqc.common import *
from saqc.core.register import FUNC_MAP
from saqc.core.lib import SaQCFunction
from saqc.lib.types import FreqString, ColumnName, IntegerWindow
-from saqc.flagger import (
- CategoricalFlagger,
- SimpleFlagger,
- DmpFlagger,
-)
+from saqc.flagger import Flagger, initFlagsLike
TESTNODATA = (np.nan, -9999)
-
-
-TESTFLAGGER = (
- CategoricalFlagger(["NIL", "GOOD", "BAD"]),
- SimpleFlagger(),
- DmpFlagger(),
-)
+TESTFLAGGER = (Flagger(),)
def flagAll(data, field, flagger, **kwargs):
# NOTE: remember to rename flag -> flag_values
- return data, flagger.setFlags(field=field, flag=flagger.BAD)
+ flagger.copy()
+ flagger[:, field] = BAD
+ return data, flagger
def initData(cols=2, start_date="2017-01-01", end_date="2017-12-31", freq=None, rows=None):
@@ -125,10 +118,10 @@ def flaggers(draw, data):
initialize a flagger and set some flags
"""
# flagger = draw(sampled_from(TESTFLAGGER)).initFlags(data)
- flagger = draw(sampled_from([SimpleFlagger()])).initFlags(data)
+ flagger = initFlagsLike(data)
for col, srs in data.items():
loc_st = lists(sampled_from(sorted(srs.index)), unique=True, max_size=len(srs)-1)
- flagger = flagger.setFlags(field=col, loc=draw(loc_st))
+ flagger[draw(loc_st), col] = BAD
return flagger
diff --git a/test/core/test_core.py b/test/core/test_core.py
index d409eb6a8..5527f2ee2 100644
--- a/test/core/test_core.py
+++ b/test/core/test_core.py
@@ -8,10 +8,12 @@ import numpy as np
import pandas as pd
-from saqc import SaQC, register
+from saqc.common import *
+from saqc.flagger import Flagger, initFlagsLike
from saqc.funcs import flagRange
from saqc.lib import plotting as splot
from test.common import initData, TESTFLAGGER, flagAll
+from saqc import SaQC, register
# no logging output needed here
@@ -31,13 +33,12 @@ def data():
@pytest.fixture
-def flags(flagger, data, optional):
+def flags(data, optional):
if not optional:
- return flagger.initFlags(data[data.columns[::2]])._flags
+ return initFlagsLike(data[data.columns[::2]]).toDios()
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_errorHandling(data, flagger):
+def test_errorHandling(data):
@register(masking='field')
def raisingFunc(data, field, flagger, **kwargs):
@@ -47,18 +48,17 @@ def test_errorHandling(data, flagger):
for policy in ["ignore", "warn"]:
# NOTE: should not fail, that's all we are testing here
- SaQC(flagger, data, error_policy=policy).raisingFunc(var1).getResult()
+ SaQC(data, error_policy=policy).raisingFunc(var1).getResult()
with pytest.raises(TypeError):
- SaQC(flagger, data, error_policy='raise').raisingFunc(var1).getResult()
+ SaQC(data, error_policy='raise').raisingFunc(var1).getResult()
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_duplicatedVariable(flagger):
+def test_duplicatedVariable():
data = initData(1)
var1 = data.columns[0]
- pdata, pflags = SaQC(flagger, data).flagtools.flagDummy(var1).getResult()
+ pdata, pflags = SaQC(data).flagtools.flagDummy(var1).getResult()
if isinstance(pflags.columns, pd.MultiIndex):
cols = pflags.columns.get_level_values(0).drop_duplicates()
@@ -67,8 +67,7 @@ def test_duplicatedVariable(flagger):
assert (pflags.columns == [var1]).all()
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_sourceTarget(flagger):
+def test_sourceTarget():
"""
test implicit assignments
"""
@@ -76,32 +75,29 @@ def test_sourceTarget(flagger):
var1 = data.columns[0]
target = "new"
- pdata, pflagger = SaQC(flagger, data).flagAll(field=var1, target=target).getResult(raw=True)
- pflags = pflagger.isFlagged()
+ pdata, pflagger = SaQC(data).flagAll(field=var1, target=target).getResult(raw=True)
assert (pdata[var1] == pdata[target]).all(axis=None)
- assert (pflags[var1] == False).all(axis=None)
- assert (pflags[target] == True).all(axis=None)
+ assert all(pflagger[var1] == UNFLAGGED)
+ assert all(pflagger[target] > UNFLAGGED)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
@pytest.mark.parametrize("optional", OPTIONAL)
-def test_dtypes(data, flagger, flags):
+def test_dtypes(data, flags):
"""
Test if the categorical dtype is preserved through the core functionality
"""
- flagger = flagger.initFlags(data)
- flags = flagger.getFlags()
+ flagger = initFlagsLike(data)
+ flags = flagger.toDios()
var1, var2 = data.columns[:2]
- pdata, pflagger = SaQC(flagger, data, flags=flags).flagAll(var1).flagAll(var2).getResult(raw=True)
+ pdata, pflagger = SaQC(data, flags=flags).flagAll(var1).flagAll(var2).getResult(raw=True)
- pflags = pflagger.getFlags()
- assert dict(flags.dtypes) == dict(pflags.dtypes)
+ for c in pflagger.columns:
+ assert pflagger[c].dtype == flagger[c].dtype
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_plotting(data, flagger):
+def test_plotting(data):
"""
Test if the plotting code runs, does not show any plot.
@@ -110,9 +106,9 @@ def test_plotting(data, flagger):
"""
pytest.importorskip("matplotlib", reason="requires matplotlib")
field, *_ = data.columns
- flagger = flagger.initFlags(data)
- _, flagger_range = flagRange(data, field, flagger, min=10, max=90, flag=flagger.BAD)
- data_new, flagger_range = flagRange(data, field, flagger_range, min=40, max=60, flag=flagger.GOOD)
+ flagger = initFlagsLike(data)
+ _, flagger_range = flagRange(data, field, flagger, min=10, max=90, flag=BAD)
+ data_new, flagger_range = flagRange(data, field, flagger_range, min=40, max=60, flag=DOUBT)
splot._interactive = False
splot._plotSingleVariable(data, data_new, flagger, flagger_range, sources=[], targets=[data_new.columns[0]])
splot._plotMultipleVariables(data, data_new, flagger, flagger_range, targets=data_new.columns)
diff --git a/test/core/test_core_new.py b/test/core/test_creation.py
similarity index 100%
rename from test/core/test_core_new.py
rename to test/core/test_creation.py
diff --git a/test/core/test_masking.py b/test/core/test_masking.py
index 4d285eabf..48207621e 100644
--- a/test/core/test_masking.py
+++ b/test/core/test_masking.py
@@ -12,7 +12,8 @@ from hypothesis.strategies import (
sampled_from,
)
-from saqc.core.core import _maskData, _unmaskData
+from saqc.common import *
+from saqc.core.register import _maskData, _unmaskData
from test.common import dataFieldFlagger, MAX_EXAMPLES
diff --git a/test/core/test_reader.py b/test/core/test_reader.py
index ef5b1c841..ce8438eff 100644
--- a/test/core/test_reader.py
+++ b/test/core/test_reader.py
@@ -12,7 +12,6 @@ from saqc.core.config import Fields as F
from test.common import initData, writeIO
from saqc.core.core import SaQC
-from saqc.flagger import SimpleFlagger
from saqc.core.register import FUNC_MAP, register
@@ -29,7 +28,7 @@ def test_packagedConfig():
data_path = path / "data.csv"
data = pd.read_csv(data_path, index_col=0, parse_dates=True,)
- saqc = SaQC(SimpleFlagger(), dios.DictOfSeries(data)).readConfig(config_path)
+ saqc = SaQC(dios.DictOfSeries(data)).readConfig(config_path)
saqc.getResult()
@@ -46,7 +45,7 @@ def test_variableRegex(data):
for regex, expected in tests:
fobj = writeIO(header + "\n" + f"{regex} ; flagtools.flagDummy()")
- saqc = SaQC(SimpleFlagger(), data).readConfig(fobj)
+ saqc = SaQC(data).readConfig(fobj)
expansion = saqc._expandFields(saqc._to_call[0][0], saqc._to_call[0][2], data.columns)
result = [s.field for s, _ in expansion]
assert np.all(result == expected)
@@ -60,7 +59,7 @@ def test_inlineComments(data):
{F.VARNAME} ; {F.TEST} ; {F.PLOT}
pre2 ; flagtools.flagDummy() # test ; False # test
"""
- saqc = SaQC(SimpleFlagger(), data).readConfig(writeIO(config))
+ saqc = SaQC(data).readConfig(writeIO(config))
_, control, func = saqc._to_call[0]
assert control.plot is False
assert func.func == FUNC_MAP["flagtools.flagDummy"].func
@@ -78,7 +77,7 @@ def test_configReaderLineNumbers(data):
SM1 ; flagtools.flagDummy()
"""
- saqc = SaQC(SimpleFlagger(), data).readConfig(writeIO(config))
+ saqc = SaQC(data).readConfig(writeIO(config))
result = [c.lineno for _, c, _ in saqc._to_call]
expected = [3, 4, 5, 9]
assert result == expected
@@ -100,7 +99,7 @@ def test_configFile(data):
SM1;flagtools.flagDummy()
"""
- SaQC(SimpleFlagger(), data).readConfig(writeIO(config))
+ SaQC(data).readConfig(writeIO(config))
def test_configChecks(data):
@@ -122,7 +121,7 @@ def test_configChecks(data):
for test, expected in tests:
fobj = writeIO(header + "\n" + test)
with pytest.raises(expected):
- SaQC(SimpleFlagger(), data).readConfig(fobj).getResult()
+ SaQC(data).readConfig(fobj).getResult()
def test_supportedArguments(data):
@@ -151,4 +150,4 @@ def test_supportedArguments(data):
for test in tests:
fobj = writeIO(header + "\n" + test)
- SaQC(SimpleFlagger(), data).readConfig(fobj)
+ SaQC(data).readConfig(fobj)
diff --git a/test/flagger/test_dmpflagger.py b/test/flagger/test_dmpflagger.py
index b1a9c1b73..677f54cbe 100644
--- a/test/flagger/test_dmpflagger.py
+++ b/test/flagger/test_dmpflagger.py
@@ -8,7 +8,9 @@ import pandas as pd
import pytest
from test.common import initData
-from saqc.flagger import DmpFlagger
+
+DmpFlagger = NotImplemented
+pytest.skip("DmpFlagger is deprecated.", allow_module_level=True)
@pytest.fixture
diff --git a/test/flagger/test_positionalflagger.py b/test/flagger/test_positionalflagger.py
index 9875a7c74..45506a070 100644
--- a/test/flagger/test_positionalflagger.py
+++ b/test/flagger/test_positionalflagger.py
@@ -6,8 +6,9 @@ import pytest
import numpy as np
from test.common import initData
-from saqc.flagger import PositionalFlagger
+PositionalFlagger = NotImplemented
+pytest.skip("PositionalFlagger is deprecated.", allow_module_level=True)
@pytest.fixture
def data():
diff --git a/test/funcs/conftest.py b/test/funcs/conftest.py
index 1fd4685e6..abecdd3f2 100644
--- a/test/funcs/conftest.py
+++ b/test/funcs/conftest.py
@@ -16,7 +16,6 @@ def char_dict():
}
-
@pytest.fixture
def course_1(char_dict):
# MONOTONOUSLY ASCENDING/DESCENDING
@@ -24,23 +23,22 @@ def course_1(char_dict):
# the resulting drop/raise per value equals: (peak_level - initial_level) / (0.5*(periods-2))
# periods number better be even!
def fix_funk(
- freq="10min",
- periods=10,
- initial_level=0,
- peak_level=10,
- initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
- char_dict=char_dict,
- name='data'
+ freq="10min",
+ periods=10,
+ initial_level=0,
+ peak_level=10,
+ initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
+ char_dict=char_dict,
+ name='data'
):
-
t_index = pd.date_range(initial_index, freq=freq, periods=periods)
left = np.linspace(initial_level, peak_level, int(np.floor(len(t_index) / 2)))
right = np.linspace(peak_level, initial_level, int(np.ceil(len(t_index) / 2)))
s = pd.Series(np.append(left, right), index=t_index)
- char_dict["raise"] = s.index[1 : int(np.floor(len(t_index) / 2))]
- char_dict["drop"] = s.index[int(np.floor(len(t_index) / 2) + 1) :]
- char_dict["peak"] = s.index[int(np.floor(len(t_index) / 2)) - 1 : int(np.floor(len(t_index) / 2)) + 1]
+ char_dict["raise"] = s.index[1: int(np.floor(len(t_index) / 2))]
+ char_dict["drop"] = s.index[int(np.floor(len(t_index) / 2) + 1):]
+ char_dict["peak"] = s.index[int(np.floor(len(t_index) / 2)) - 1: int(np.floor(len(t_index) / 2)) + 1]
data = DictOfSeries(data=s, columns=[name])
return data, char_dict
@@ -55,13 +53,13 @@ def course_2(char_dict):
# one "anomalous" or "outlierish" value of magnitude "out_val" at position "periods/2"
# number of periods better be even!
def fix_funk(
- freq="10min",
- periods=10,
- initial_level=0,
- final_level=2,
- out_val=5,
- initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
- char_dict=char_dict,
+ freq="10min",
+ periods=10,
+ initial_level=0,
+ final_level=2,
+ out_val=5,
+ initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
+ char_dict=char_dict,
):
t_index = pd.date_range(initial_index, freq=freq, periods=periods)
data = np.linspace(initial_level, final_level, int(np.floor(len(t_index))))
@@ -88,21 +86,18 @@ def course_test(char_dict):
# Test function for pattern detection - same as test pattern for first three values, than constant function
def fix_funk(freq='1 D',
initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0), out_val=5, char_dict=char_dict):
-
t_index = pd.date_range(initial_index, freq=freq, periods=100)
data = pd.Series(data=0, index=t_index)
data.iloc[2] = out_val
data.iloc[3] = out_val
-
data = DictOfSeries(data=data, columns=['data'])
return data, char_dict
return fix_funk
-
@pytest.fixture
def course_3(char_dict):
# CROWD IN A PIT/CROWD ON A SUMMIT
@@ -113,15 +108,15 @@ def course_3(char_dict):
# number of periods better be even!
# chrowd_size * crowd_spacing better be less then freq[minutes].
def fix_funk(
- freq="10min",
- periods=10,
- initial_level=0,
- final_level=2,
- out_val=-5,
- initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
- char_dict=char_dict,
- crowd_size=5,
- crowd_spacing=1,
+ freq="10min",
+ periods=10,
+ initial_level=0,
+ final_level=2,
+ out_val=-5,
+ initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
+ char_dict=char_dict,
+ crowd_size=5,
+ crowd_spacing=1,
):
t_index = pd.date_range(initial_index, freq=freq, periods=periods)
@@ -158,19 +153,18 @@ def course_4(char_dict):
# of periods better be even!
def fix_funk(
- freq="10min",
- periods=10,
- base_level=0,
- out_val=5,
- initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
- char_dict=char_dict,
+ freq="10min",
+ periods=10,
+ base_level=0,
+ out_val=5,
+ initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
+ char_dict=char_dict,
):
-
t_index = pd.date_range(initial_index, freq=freq, periods=periods)
data = pd.Series(data=base_level, index=t_index)
- data[int(len(t_index) / 2) :: 2] = out_val
- char_dict["raise"] = t_index[int(len(t_index) / 2) :: 2]
- char_dict["return"] = t_index[int((len(t_index) / 2) + 1) :: 2]
+ data[int(len(t_index) / 2):: 2] = out_val
+ char_dict["raise"] = t_index[int(len(t_index) / 2):: 2]
+ char_dict["return"] = t_index[int((len(t_index) / 2) + 1):: 2]
data = DictOfSeries(data=data, columns=["data"])
return data, char_dict
@@ -187,13 +181,13 @@ def course_5(char_dict):
# periods better be greater 5
def fix_funk(
- freq="10min",
- periods=10,
- nan_slice=slice(0, None, 5),
- initial_level=0,
- final_level=10,
- initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
- char_dict=char_dict,
+ freq="10min",
+ periods=10,
+ nan_slice=slice(0, None, 5),
+ initial_level=0,
+ final_level=10,
+ initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
+ char_dict=char_dict,
):
t_index = pd.date_range(initial_index, freq=freq, periods=periods)
values = np.linspace(initial_level, final_level, periods)
@@ -205,5 +199,3 @@ def course_5(char_dict):
return data, char_dict
return fix_funk
-
-
diff --git a/test/funcs/test_constants_detection.py b/test/funcs/test_constants_detection.py
index 75dab02ae..b7cabb50e 100644
--- a/test/funcs/test_constants_detection.py
+++ b/test/funcs/test_constants_detection.py
@@ -6,7 +6,7 @@ import numpy as np
from saqc.funcs.constants import flagConstants, flagByVariance
-from test.common import TESTFLAGGER, initData
+from test.common import initData, initFlagsLike, BAD
@pytest.fixture
@@ -16,23 +16,21 @@ def data():
return constants_data
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_constants_flagBasic(data, flagger):
+def test_constants_flagBasic(data):
expected = np.arange(5, 22)
field, *_ = data.columns
- flagger = flagger.initFlags(data)
- data, flagger_result = flagConstants(data, field, flagger, window="15Min", thresh=0.1, )
- flags = flagger_result.getFlags(field)
- assert np.all(flags[expected] == flagger.BAD)
+ flagger = initFlagsLike(data)
+ data, flagger_result = flagConstants(data, field, flagger, window="15Min", thresh=0.1, flag=BAD)
+ flags = flagger_result[field]
+ assert np.all(flags[expected] == BAD)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_constants_flagVarianceBased(data, flagger):
+def test_constants_flagVarianceBased(data):
expected = np.arange(5, 25)
field, *_ = data.columns
- flagger = flagger.initFlags(data)
- data, flagger_result1 = flagByVariance(data, field, flagger, window="1h")
+ flagger = initFlagsLike(data)
+ data, flagger_result1 = flagByVariance(data, field, flagger, window="1h", flag=BAD)
- flag_result1 = flagger_result1.getFlags(field)
- test_sum = (flag_result1[expected] == flagger.BAD).sum()
+ flag_result1 = flagger_result1[field]
+ test_sum = (flag_result1[expected] == BAD).sum()
assert test_sum == len(expected)
diff --git a/test/funcs/test_functions.py b/test/funcs/test_functions.py
index a47331cd0..2a466df14 100644
--- a/test/funcs/test_functions.py
+++ b/test/funcs/test_functions.py
@@ -6,6 +6,8 @@ import pandas as pd
import numpy as np
import dios
+from saqc.common import *
+from saqc.flagger import Flagger, initFlagsLike
from saqc.funcs.drift import flagDriftFromNorm, flagDriftFromReference, flagDriftFromScaledNorm
from saqc.funcs.outliers import flagCrossStatistic, flagRange
from saqc.funcs.flagtools import flagManual, forceFlags, clearFlags
@@ -15,8 +17,6 @@ from saqc.funcs.breaks import flagIsolated
from test.common import initData, TESTFLAGGER
-
-
@pytest.fixture
def data():
return initData(cols=1, start_date="2016-01-01", end_date="2018-12-31", freq="1D")
@@ -27,89 +27,88 @@ def field(data):
return data.columns[0]
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_flagRange(data, field, flagger):
+def test_flagRange(data, field):
min, max = 10, 90
- flagger = flagger.initFlags(data)
- data, flagger = flagRange(data, field, flagger, min=min, max=max)
- flagged = flagger.isFlagged(field)
+ flagger = initFlagsLike(data)
+ data, flagger = flagRange(data, field, flagger, min=min, max=max, flag=BAD)
+ flagged = flagger[field] > UNFLAGGED
expected = (data[field] < min) | (data[field] > max)
- assert (flagged == expected).all()
+ assert all(flagged == expected)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_flagSesonalRange(data, field, flagger):
+def test_flagSesonalRange(data, field):
# prepare
data.iloc[::2] = 0
data.iloc[1::2] = 50
nyears = len(data[field].index.year.unique())
tests = [
- ({"min": 1, "max": 100, "startmonth": 7, "startday": 1, "endmonth": 8, "endday": 31,}, 31 * 2 * nyears // 2,),
- ({"min": 1, "max": 100, "startmonth": 12, "startday": 16, "endmonth": 1, "endday": 15,}, 31 * nyears // 2 + 1,),
+ ({"min": 1, "max": 100, "startmonth": 7, "startday": 1, "endmonth": 8, "endday": 31, }, 31 * 2 * nyears // 2,),
+ (
+ {"min": 1, "max": 100, "startmonth": 12, "startday": 16, "endmonth": 1, "endday": 15, }, 31 * nyears // 2 + 1,),
]
for test, expected in tests:
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
newfield = f"{field}_masked"
start = f"{test['startmonth']:02}-{test['startday']:02}T00:00:00"
end = f"{test['endmonth']:02}-{test['endday']:02}T00:00:00"
data, flagger = copy(data, field, flagger, field + "_masked")
- data, flagger = mask(data, newfield, flagger, mode='periodic', period_start=start, period_end=end,
- include_bounds=True)
- data, flagger = flagRange(data, newfield, flagger, min=test['min'], max=test['max'])
- data, flagger = reindexFlags(data, field, flagger, method='match', source=newfield)
+ data, flagger = mask(
+ data, newfield, flagger,
+ mode='periodic', period_start=start, period_end=end, include_bounds=True, flag=BAD
+ )
+ data, flagger = flagRange(data, newfield, flagger, min=test['min'], max=test['max'], flag=BAD)
+ data, flagger = reindexFlags(data, field, flagger, method='match', source=newfield, flag=BAD)
data, flagger = drop(data, newfield, flagger)
- flagged = flagger.isFlagged(field)
+ flagged = flagger[field] > UNFLAGGED
assert flagged.sum() == expected
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_clearFlags(data, field, flagger):
- flagger = flagger.initFlags(data)
- flags_orig = flagger.getFlags()
- flags_set = flagger.setFlags(field, flag=flagger.BAD).getFlags()
+def test_clearFlags(data, field):
+ flagger = initFlagsLike(data)
+ flagger[:, field] = BAD
+ assert all(flagger[field] == BAD)
+
_, flagger = clearFlags(data, field, flagger)
- flags_cleared = flagger.getFlags()
- assert (flags_orig != flags_set).all(None)
- assert (flags_orig == flags_cleared).all(None)
+ assert all(flagger[field] == UNFLAGGED)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_forceFlags(data, flagger):
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
- flags_orig = flagger.setFlags(field).getFlags(field)
- _, flagger = forceFlags(data, field, flagger, flag=flagger.GOOD)
- flags_forced = flagger.getFlags(field)
- assert np.all(flags_orig != flags_forced)
+def test_forceFlags(data, field):
+ flagger = initFlagsLike(data)
+ flagger[:, field] = BAD
+ assert all(flagger[field] == BAD)
+ _, flagger = forceFlags(data, field, flagger, flag=DOUBT)
+ assert all(flagger[field] == DOUBT)
+
+
+# todo: @luenensc: i dont get the test -- palmb
+def test_flagIsolated(data, field):
+ flagger = initFlagsLike(data)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_flagIsolated(data, flagger):
- field = data.columns[0]
data.iloc[1:3, 0] = np.nan
data.iloc[4:5, 0] = np.nan
data.iloc[11:13, 0] = np.nan
data.iloc[15:17, 0] = np.nan
- flagger = flagger.initFlags(data)
+
s = data[field].iloc[5:6]
- flagger = flagger.setFlags(field, loc=s)
+ flagger[s.index, field] = BAD
- _, flagger_result = flagIsolated(data, field, flagger, group_window="1D", gap_window="2.1D")
+ _, flagger_result = flagIsolated(data, field, flagger, group_window="1D", gap_window="2.1D", flag=BAD)
- assert flagger_result.isFlagged(field)[slice(3, 6, 2)].all()
+ assert flagger_result[field][slice(3, 6, 2)].all()
data, flagger_result = flagIsolated(
- data, field, flagger_result, group_window="2D", gap_window="2.1D", continuation_range="1.1D",
+ data, field, flagger_result,
+ group_window="2D", gap_window="2.1D", continuation_range="1.1D", flag=BAD
)
- assert flagger_result.isFlagged(field)[[3, 5, 13, 14]].all()
+ assert flagger_result[field][[3, 5, 13, 14]].all()
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
@pytest.mark.parametrize("dat", [pytest.lazy_fixture("course_2")])
-def test_flagCrossScoring(dat, flagger):
+def test_flagCrossScoring(dat):
data1, characteristics = dat(initial_level=0, final_level=0, out_val=0)
data2, characteristics = dat(initial_level=0, final_level=0, out_val=10)
field = "dummy"
@@ -118,17 +117,15 @@ def test_flagCrossScoring(dat, flagger):
s1 = pd.Series(data=s1.values, index=s1.index)
s2 = pd.Series(data=s2.values, index=s1.index)
data = dios.DictOfSeries([s1, s2], columns=["data1", "data2"])
- flagger = flagger.initFlags(data)
- _, flagger_result = flagCrossStatistic(data, field, flagger, fields=fields, thresh=3, cross_stat=np.mean)
+ flagger = initFlagsLike(data)
+ _, flagger_result = flagCrossStatistic(data, field, flagger, fields=fields, thresh=3, cross_stat=np.mean, flag=BAD)
for field in fields:
- isflagged = flagger_result.isFlagged(field)
+ isflagged = flagger_result[field] > UNFLAGGED
assert isflagged[characteristics["raise"]].all()
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_flagManual(data, flagger):
- field = data.columns[0]
- flagger = flagger.initFlags(data)
+def test_flagManual(data, field):
+ flagger = initFlagsLike(data)
args = data, field, flagger
dat = data[field]
@@ -139,20 +136,20 @@ def test_flagManual(data, flagger):
shrinked = mdata.loc[index_exp.union(mdata.iloc[[1, 2, 3, 4, 600, 601]].index)]
kwargs_list = [
- dict(mdata=mdata, mflag="a", method="plain"),
- dict(mdata=mdata.to_list(), mflag="a", method="plain"),
- dict(mdata=mdata, mflag="a", method="ontime"),
- dict(mdata=shrinked, mflag="a", method="ontime"),
+ dict(mdata=mdata, mflag="a", method="plain", flag=BAD),
+ dict(mdata=mdata.to_list(), mflag="a", method="plain", flag=BAD),
+ dict(mdata=mdata, mflag="a", method="ontime", flag=BAD),
+ dict(mdata=shrinked, mflag="a", method="ontime", flag=BAD),
]
for kw in kwargs_list:
_, fl = flagManual(*args, **kw)
- isflagged = fl.isFlagged(field)
+ isflagged = fl[field] > UNFLAGGED
assert isflagged[isflagged].index.equals(index_exp)
# flag not exist in mdata
- _, fl = flagManual(*args, mdata=mdata, mflag="i do not exist", method="ontime")
- isflagged = fl.isFlagged(field)
+ _, fl = flagManual(*args, mdata=mdata, mflag="i do not exist", method="ontime", flag=BAD)
+ isflagged = fl[field] > UNFLAGGED
assert isflagged[isflagged].index.equals(pd.DatetimeIndex([]))
# check right-open / ffill
@@ -179,9 +176,10 @@ def test_flagManual(data, flagger):
expected.loc[dat.index[-1]] = 1
expected = expected.astype(bool)
- _, fl = flagManual(*args, mdata=mdata, mflag=1, method="right-open")
- isflagged = fl.isFlagged(field)
+ _, fl = flagManual(*args, mdata=mdata, mflag=1, method="right-open", flag=BAD)
+ isflagged = fl[field] > UNFLAGGED
last = expected.index[0]
+
for curr in expected.index[1:]:
expected_value = mdata[last]
# datetime slicing is inclusive !
@@ -194,10 +192,11 @@ def test_flagManual(data, flagger):
# check left-open / bfill
expected.loc[dat.index[-1]] = 0 # this time the last is False
- _, fl = flagManual(*args, mdata=mdata, mflag=1, method="left-open")
- isflagged = fl.isFlagged(field)
+ _, fl = flagManual(*args, mdata=mdata, mflag=1, method="left-open", flag=BAD)
+ isflagged = fl[field] > UNFLAGGED
last = expected.index[0]
assert isflagged[last] == expected[last]
+
for curr in expected.index[1:]:
expected_value = mdata[curr]
# datetime slicing is inclusive !
@@ -206,24 +205,40 @@ def test_flagManual(data, flagger):
assert (chunk == expected_value).all()
last = curr
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
+
@pytest.mark.parametrize("dat", [pytest.lazy_fixture("course_1")])
-def test_flagDriftFromNormal(dat, flagger):
+def test_flagDriftFromNormal(dat):
data = dat(periods=200, peak_level=5, name='d1')[0]
data['d2'] = dat(periods=200, peak_level=10, name='d2')[0]['d2']
data['d3'] = dat(periods=200, peak_level=100, name='d3')[0]['d3']
data['d4'] = 3 + 4 * data['d1']
data['d5'] = 3 + 4 * data['d1']
- flagger = flagger.initFlags(data)
- data_norm, flagger_norm = flagDriftFromNorm(data, 'dummy', flagger, ['d1', 'd2', 'd3'], segment_freq="200min",
- norm_spread=5)
+ flagger = initFlagsLike(data)
+ data_norm, flagger_norm = flagDriftFromNorm(
+ data, 'dummy', flagger,
+ ['d1', 'd2', 'd3'],
+ segment_freq="200min",
+ norm_spread=5,
+ flag=BAD,
+ )
- data_ref, flagger_ref = flagDriftFromReference(data, 'd1', flagger, ['d1', 'd2', 'd3'], segment_freq="3D",
- thresh=20)
+ data_ref, flagger_ref = flagDriftFromReference(
+ data, 'd1', flagger,
+ ['d1', 'd2', 'd3'],
+ segment_freq="3D",
+ thresh=20,
+ flag=BAD,
+ )
- data_scale, flagger_scale = flagDriftFromScaledNorm(data, 'dummy', flagger, ['d1', 'd3'], ['d4', 'd5'], segment_freq="3D",
- thresh=20, norm_spread=5)
- assert flagger_norm.isFlagged()['d3'].all()
- assert flagger_ref.isFlagged()['d3'].all()
- assert flagger_scale.isFlagged()['d3'].all()
+ data_scale, flagger_scale = flagDriftFromScaledNorm(
+ data, 'dummy', flagger,
+ ['d1', 'd3'], ['d4', 'd5'],
+ segment_freq="3D",
+ thresh=20,
+ norm_spread=5,
+ flag=BAD,
+ )
+ assert all(flagger_norm['d3'] > UNFLAGGED)
+ assert all(flagger_ref['d3'] > UNFLAGGED)
+ assert all(flagger_scale['d3'] > UNFLAGGED)
diff --git a/test/funcs/test_generic_api_functions.py b/test/funcs/test_generic_api_functions.py
index 8d200034f..950dbfd7f 100644
--- a/test/funcs/test_generic_api_functions.py
+++ b/test/funcs/test_generic_api_functions.py
@@ -6,16 +6,12 @@ import pytest
import numpy as np
import pandas as pd
-from dios import DictOfSeries
-
-from test.common import TESTFLAGGER, TESTNODATA, initData, writeIO, flagAll
-from saqc.core.visitor import ConfigFunctionParser
-from saqc.core.config import Fields as F
+from saqc.common import *
from saqc.core.register import register
-from saqc import SaQC, SimpleFlagger
-from saqc.funcs.generic import _execGeneric
+from saqc import SaQC
from saqc.funcs.tools import mask
+from test.common import initData, flagAll
register(masking='field')(flagAll)
@@ -25,38 +21,34 @@ def data():
return initData()
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_addFieldFlagGeneric(data, flagger):
- saqc = SaQC(data=data, flagger=flagger)
+def test_addFieldFlagGeneric(data):
+ saqc = SaQC(data=data)
- data, flags = saqc.generic.flag(
- "tmp1",
- func=lambda var1: pd.Series(False, index=data[var1.name].index)
- ).getResult()
- assert "tmp1" in flags.columns and "tmp1" not in data
+ func = lambda var1: pd.Series(False, index=data[var1.name].index)
+ data, flagger = saqc.generic.flag("tmp1", func, flag=BAD).getResult()
+ assert "tmp1" in flagger.columns and "tmp1" not in data
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_addFieldProcGeneric(data, flagger):
- saqc = SaQC(data=data, flagger=flagger)
+def test_addFieldProcGeneric(data):
+ saqc = SaQC(data=data)
- data, flagger = saqc.generic.process("tmp1", func=lambda: pd.Series([])).getResult(raw=True)
+ func = lambda: pd.Series([])
+ data, flagger = saqc.generic.process("tmp1", func, flag=BAD ).getResult(raw=True)
assert "tmp1" in data.columns and data["tmp1"].empty
- data, flagger = saqc.generic.process("tmp2", func=lambda var1, var2: var1 + var2).getResult()
+ func = lambda var1, var2: var1 + var2
+ data, flagger = saqc.generic.process("tmp2", func, flag=BAD).getResult()
assert "tmp2" in data.columns and (data["tmp2"] == data["var1"] + data["var2"]).all(axis=None)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_mask(data, flagger):
-
- saqc = SaQC(data=data, flagger=flagger)
+def test_mask(data):
+ saqc = SaQC(data=data)
data_org = data.copy(deep=True)
mean = data["var1"] / 2
- data, _ = saqc.generic.process("var1", lambda var1: mask(var1 < mean)).getResult()
+ data, _ = saqc.generic.process("var1", lambda var1: mask(var1 < mean), flag=BAD).getResult()
assert ((data["var1"].isna()) == (data_org["var1"] < 10) & data_org["var1"].isna()).all(axis=None)
- data, flags = saqc.generic.process("tmp", lambda var1: mask(var1 < mean)).getResult()
- assert ("tmp" in data.columns) and ("tmp" in flags.columns)
+ data, flagger = saqc.generic.process("tmp", lambda var1: mask(var1 < mean), flag=BAD).getResult()
+ assert ("tmp" in data.columns) and ("tmp" in flagger.columns)
assert ((data["tmp"].isna()) == (data_org["var1"] < 10) & data_org["var1"].isna()).all(axis=None)
diff --git a/test/funcs/test_generic_config_functions.py b/test/funcs/test_generic_config_functions.py
index 2a1e8a14c..7677c3c27 100644
--- a/test/funcs/test_generic_config_functions.py
+++ b/test/funcs/test_generic_config_functions.py
@@ -10,10 +10,13 @@ import pandas as pd
from dios import DictOfSeries
from test.common import TESTFLAGGER, TESTNODATA, initData, writeIO
+
+from saqc.common import *
+from saqc.flagger import Flagger, initFlagsLike
from saqc.core.visitor import ConfigFunctionParser
from saqc.core.config import Fields as F
from saqc.core.register import register
-from saqc import SaQC, SimpleFlagger
+from saqc import SaQC
from saqc.funcs.generic import _execGeneric
@@ -68,13 +71,12 @@ def test_syntaxError(flagger):
_compileGeneric(f"flag(func={test})", flagger)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_typeError(flagger):
-
+def test_typeError():
"""
test that forbidden constructs actually throw an error
TODO: find a few more cases or get rid of the test
"""
+ flagger = Flagger()
# : think about cases that should be forbidden
tests = ("lambda x: x * 2",)
@@ -84,9 +86,8 @@ def test_typeError(flagger):
_compileGeneric(f"generic.flag(func={test})", flagger)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_comparisonOperators(data, flagger):
- flagger = flagger.initFlags(data)
+def test_comparisonOperators(data):
+ flagger = initFlagsLike(data)
var1, var2, *_ = data.columns
this = var1
@@ -107,7 +108,7 @@ def test_comparisonOperators(data, flagger):
@pytest.mark.parametrize("flagger", TESTFLAGGER)
def test_arithmeticOperators(data, flagger):
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
var1, *_ = data.columns
this = data[var1]
@@ -128,7 +129,7 @@ def test_arithmeticOperators(data, flagger):
@pytest.mark.parametrize("flagger", TESTFLAGGER)
def test_nonReduncingBuiltins(data, flagger):
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
var1, *_ = data.columns
this = var1
mean = data[var1].mean()
@@ -151,7 +152,7 @@ def test_nonReduncingBuiltins(data, flagger):
def test_reduncingBuiltins(data, flagger, nodata):
data.loc[::4] = nodata
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
var1 = data.columns[0]
this = data.iloc[:, 0]
@@ -195,7 +196,7 @@ def test_bitOps(data, flagger, nodata):
var1, var2, *_ = data.columns
this = var1
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
tests = [
("~(this > mean(this))", ~(data[this] > np.nanmean(data[this]))),
@@ -209,19 +210,18 @@ def test_bitOps(data, flagger, nodata):
assert np.all(result == expected)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_isflagged(data, flagger):
+def test_isflagged(data):
var1, var2, *_ = data.columns
-
- flagger = flagger.initFlags(data).setFlags(var1, loc=data[var1].index[::2], flag=flagger.BAD)
+ flagger = initFlagsLike(data)
+ flagger[data[var1].index[::2], var1] = BAD
tests = [
- (f"isflagged({var1})", flagger.isFlagged(var1)),
- (f"isflagged({var1}, flag=BAD)", flagger.isFlagged(var1, flag=flagger.BAD, comparator=">=")),
- (f"isflagged({var1}, UNFLAGGED, '==')", flagger.isFlagged(var1, flag=flagger.UNFLAGGED, comparator="==")),
- (f"~isflagged({var2})", ~flagger.isFlagged(var2)),
- (f"~({var2}>999) & (~isflagged({var2}))", ~(data[var2] > 999) & (~flagger.isFlagged(var2))),
+ (f"isflagged({var1})", flagger[var1] > UNFLAGGED),
+ (f"isflagged({var1}, flag=BAD)", flagger[var1] >= BAD),
+ (f"isflagged({var1}, UNFLAGGED, '==')", flagger[var1] == UNFLAGGED),
+ (f"~isflagged({var2})", ~(flagger[var2] > UNFLAGGED)),
+ (f"~({var2}>999) & (~isflagged({var2}))", ~(data[var2] > 999) & ~(flagger[var2] > UNFLAGGED)),
]
for test, expected in tests:
@@ -230,8 +230,7 @@ def test_isflagged(data, flagger):
assert np.all(result == expected)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_variableAssignments(data, flagger):
+def test_variableAssignments(data):
var1, var2, *_ = data.columns
config = f"""
@@ -241,18 +240,17 @@ def test_variableAssignments(data, flagger):
"""
fobj = writeIO(config)
- saqc = SaQC(flagger, data).readConfig(fobj)
+ saqc = SaQC(data).readConfig(fobj)
result_data, result_flagger = saqc.getResult(raw=True)
assert set(result_data.columns) == set(data.columns) | {
"dummy1",
}
- assert set(result_flagger.getFlags().columns) == set(data.columns) | {"dummy1", "dummy2"}
+ assert set(result_flagger.columns) == set(data.columns) | {"dummy1", "dummy2"}
@pytest.mark.xfail(strict=True)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_processMultiple(data_diff, flagger):
+def test_processMultiple(data_diff):
var1, var2, *_ = data_diff.columns
config = f"""
@@ -262,9 +260,9 @@ def test_processMultiple(data_diff, flagger):
"""
fobj = writeIO(config)
- saqc = SaQC(flagger, data_diff).readConfig(fobj)
+ saqc = SaQC(data_diff).readConfig(fobj)
result_data, result_flagger = saqc.getResult()
- assert len(result_data["dummy"]) == len(result_flagger.getFlags("dummy"))
+ assert len(result_data["dummy"]) == len(result_flagger["dummy"])
def test_callableArgumentsUnary(data):
@@ -274,9 +272,8 @@ def test_callableArgumentsUnary(data):
@register(masking='field')
def testFuncUnary(data, field, flagger, func, **kwargs):
data[field] = data[field].rolling(window=window).apply(func)
- return data, flagger.initFlags(data=data)
+ return data, initFlagsLike(data)
- flagger = SimpleFlagger()
var = data.columns[0]
config = f"""
@@ -291,22 +288,20 @@ def test_callableArgumentsUnary(data):
for (name, func) in tests:
fobj = writeIO(config.format(name))
- result_config, _ = SaQC(flagger, data).readConfig(fobj).getResult()
- result_api, _ = SaQC(flagger, data).testFuncUnary(var, func=func).getResult()
+ result_config, _ = SaQC(data).readConfig(fobj).getResult()
+ result_api, _ = SaQC(data).testFuncUnary(var, func=func).getResult()
expected = data[var].rolling(window=window).apply(func)
assert (result_config[var].dropna() == expected.dropna()).all(axis=None)
assert (result_api[var].dropna() == expected.dropna()).all(axis=None)
def test_callableArgumentsBinary(data):
-
- flagger = SimpleFlagger()
var1, var2 = data.columns[:2]
@register(masking='field')
def testFuncBinary(data, field, flagger, func, **kwargs):
data[field] = func(data[var1], data[var2])
- return data, flagger.initFlags(data=data)
+ return data, initFlagsLike(data)
config = f"""
{F.VARNAME} ; {F.TEST}
@@ -320,8 +315,8 @@ def test_callableArgumentsBinary(data):
for (name, func) in tests:
fobj = writeIO(config.format(name))
- result_config, _ = SaQC(flagger, data).readConfig(fobj).getResult()
- result_api, _ = SaQC(flagger, data).testFuncBinary(var1, func=func).getResult()
+ result_config, _ = SaQC(data).readConfig(fobj).getResult()
+ result_api, _ = SaQC(data).testFuncBinary(var1, func=func).getResult()
expected = func(data[var1], data[var2])
assert (result_config[var1].dropna() == expected.dropna()).all(axis=None)
assert (result_api[var1].dropna() == expected.dropna()).all(axis=None)
diff --git a/test/funcs/test_pattern_rec.py b/test/funcs/test_pattern_rec.py
index 75f3f4e4c..0763a82f0 100644
--- a/test/funcs/test_pattern_rec.py
+++ b/test/funcs/test_pattern_rec.py
@@ -7,8 +7,10 @@ import pandas as pd
from dios import dios
+from saqc.common import *
+from saqc.flagger import Flagger, initFlagsLike
from saqc.funcs.pattern import *
-from test.common import initData, TESTFLAGGER
+from test.common import initData
@pytest.fixture
@@ -21,33 +23,31 @@ def field(data):
return data.columns[0]
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_flagPattern_wavelet(flagger):
-
+@pytest.mark.skip(reason='faulty implementation - will get fixed by GL-MR191')
+def test_flagPattern_wavelet():
data = pd.Series(0, index=pd.date_range(start="2000", end='2001', freq='1d'))
data.iloc[2:4] = 7
pattern = data.iloc[1:6]
data = dios.DictOfSeries(dict(data=data, pattern_data=pattern))
+ flagger = initFlagsLike(data, name='data')
+ data, flagger = flagPatternByDTW(data, "data", flagger, ref_field="pattern_data", flag=BAD)
- flagger = flagger.initFlags(data)
- data, flagger = flagPatternByDTW(data, "data", flagger, ref_field="pattern_data")
- assert (flagger.isFlagged("data")[1:6]).all()
- assert (flagger.isFlagged("data")[:1]).any()
- assert (flagger.isFlagged("data")[7:]).any()
-
+ assert all(flagger["data"][1:6])
+ assert any(flagger["data"][:1])
+ assert any(flagger["data"][7:])
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_flagPattern_dtw(flagger):
+@pytest.mark.skip(reason='faulty implementation - will get fixed by GL-MR191')
+def test_flagPattern_dtw():
data = pd.Series(0, index=pd.date_range(start="2000", end='2001', freq='1d'))
data.iloc[2:4] = 7
pattern = data.iloc[1:6]
data = dios.DictOfSeries(dict(data=data, pattern_data=pattern))
+ flagger = initFlagsLike(data, name='data')
+ data, flagger = flagPatternByWavelet(data, "data", flagger, ref_field="pattern_data", flag=BAD)
- flagger = flagger.initFlags(data)
- data, flagger = flagPatternByWavelet(data, "data", flagger, ref_field="pattern_data")
- assert (flagger.isFlagged("data")[1:6]).all()
- assert (flagger.isFlagged("data")[:1]).any()
- assert (flagger.isFlagged("data")[7:]).any()
+ assert all(flagger["data"][1:6])
+ assert any(flagger["data"][:1])
+ assert any(flagger["data"][7:])
diff --git a/test/funcs/test_spikes_detection.py b/test/funcs/test_spikes_detection.py
index da8683479..be38370e3 100644
--- a/test/funcs/test_spikes_detection.py
+++ b/test/funcs/test_spikes_detection.py
@@ -16,6 +16,8 @@ from saqc.funcs.outliers import (
)
from test.common import TESTFLAGGER
+from saqc.common import *
+from saqc.flagger import Flagger, initFlagsLike
@pytest.fixture(scope="module")
@@ -28,30 +30,27 @@ def spiky_data():
return dios.DictOfSeries(s), flag_assertion
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_flagMad(spiky_data, flagger):
+def test_flagMad(spiky_data):
data = spiky_data[0]
field, *_ = data.columns
- flagger = flagger.initFlags(data)
- data, flagger_result = flagMAD(data, field, flagger, "1H")
- flag_result = flagger_result.getFlags(field)
- test_sum = (flag_result[spiky_data[1]] == flagger.BAD).sum()
+ flagger = initFlagsLike(data)
+ data, flagger_result = flagMAD(data, field, flagger, "1H", flag=BAD)
+ flag_result = flagger_result[field]
+ test_sum = (flag_result[spiky_data[1]] == BAD).sum()
assert test_sum == len(spiky_data[1])
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_flagSpikesBasic(spiky_data, flagger):
+def test_flagSpikesBasic(spiky_data):
data = spiky_data[0]
field, *_ = data.columns
- flagger = flagger.initFlags(data)
- data, flagger_result = flagOffset(data, field, flagger, thresh=60, tolerance=10, window="20min")
- flag_result = flagger_result.getFlags(field)
- test_sum = (flag_result[spiky_data[1]] == flagger.BAD).sum()
+ flagger = initFlagsLike(data)
+ data, flagger_result = flagOffset(data, field, flagger, thresh=60, tolerance=10, window="20min", flag=BAD)
+ flag_result = flagger_result[field]
+ test_sum = (flag_result[spiky_data[1]] == BAD).sum()
assert test_sum == len(spiky_data[1])
# see test/functs/conftest.py for the 'course_N'
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
@pytest.mark.parametrize(
"dat",
[
@@ -61,22 +60,22 @@ def test_flagSpikesBasic(spiky_data, flagger):
pytest.lazy_fixture("course_4"),
],
)
-def test_flagSpikesLimitRaise(dat, flagger):
+def test_flagSpikesLimitRaise(dat):
data, characteristics = dat()
field, *_ = data.columns
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
_, flagger_result = flagRaise(
- data, field, flagger, thresh=2, intended_freq="10min", raise_window="20min", numba_boost=False
+ data, field, flagger,
+ thresh=2, intended_freq="10min", raise_window="20min", numba_boost=False, flag=BAD
)
- assert flagger_result.isFlagged(field)[characteristics["raise"]].all()
- assert not flagger_result.isFlagged(field)[characteristics["return"]].any()
- assert not flagger_result.isFlagged(field)[characteristics["drop"]].any()
+ assert np.all(flagger_result[field][characteristics["raise"]] > UNFLAGGED)
+ assert not np.any(flagger_result[field][characteristics["return"]] > UNFLAGGED)
+ assert not np.any(flagger_result[field][characteristics["drop"]] > UNFLAGGED)
# see test/functs/conftest.py for the 'course_N'
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
@pytest.mark.parametrize("dat", [pytest.lazy_fixture("course_3")])
-def test_flagMultivarScores(dat, flagger):
+def test_flagMultivarScores(dat):
data1, characteristics = dat(periods=1000, initial_level=5, final_level=15, out_val=50)
data2, characteristics = dat(periods=1000, initial_level=20, final_level=1, out_val=30)
field = "dummy"
@@ -85,24 +84,26 @@ def test_flagMultivarScores(dat, flagger):
s1 = pd.Series(data=s1.values, index=s1.index)
s2 = pd.Series(data=s2.values, index=s1.index)
data = dios.DictOfSeries([s1, s2], columns=["data1", "data2"])
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
_, flagger_result = flagMVScores(
- data, field, flagger, fields=fields, trafo=np.log, iter_start=0.95, n_neighbors=10
+ data, field, flagger, fields=fields, trafo=np.log, iter_start=0.95, n_neighbors=10, flag=BAD
)
for field in fields:
- isflagged = flagger_result.isFlagged(field)
+ isflagged = flagger_result[field] > UNFLAGGED
assert isflagged[characteristics["raise"]].all()
assert not isflagged[characteristics["return"]].any()
assert not isflagged[characteristics["drop"]].any()
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
@pytest.mark.parametrize("dat", [pytest.lazy_fixture("course_3")])
-def test_grubbs(dat, flagger):
+def test_grubbs(dat):
data, char_dict = dat(
- freq="10min", periods=45, initial_level=0, final_level=0, crowd_size=1, crowd_spacing=3, out_val=-10
+ freq="10min", periods=45,
+ initial_level=0, final_level=0,
+ crowd_size=1, crowd_spacing=3,
+ out_val=-10,
)
- flagger = flagger.initFlags(data)
- data, result_flagger = flagByGrubbs(data, "data", flagger, winsz=20, min_periods=15)
- assert result_flagger.isFlagged("data")[char_dict["drop"]].all()
+ flagger = initFlagsLike(data)
+ data, result_flagger = flagByGrubbs(data, "data", flagger, winsz=20, min_periods=15, flag=BAD)
+ assert np.all(result_flagger["data"][char_dict["drop"]] > UNFLAGGED)
--
GitLab
From a81c3c8a21396282dbae938fd7d61e8415258275 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Thu, 4 Mar 2021 15:16:29 +0100
Subject: [PATCH 033/180] fixed plotting.py - was easy
---
saqc/lib/plotting.py | 26 ++++++++++++--------------
1 file changed, 12 insertions(+), 14 deletions(-)
diff --git a/saqc/lib/plotting.py b/saqc/lib/plotting.py
index 34b0eb7c4..c03670b5c 100644
--- a/saqc/lib/plotting.py
+++ b/saqc/lib/plotting.py
@@ -406,14 +406,14 @@ def _getDataFromVar(
All infos are projected to the data locations.
"""
var = varname
- assert var in flagger_new.flags
- flags_new: pd.Series = flagger_new.flags[var]
+ assert var in flagger_new.columns
+ flags_new: pd.Series = flagger_new[var]
plotdict = _getPlotdict(data_new, flags_new, flagger_new, var)
ref_plotdict = None
# prepare flags
- if flagger_old is not None and var in flagger_old.flags:
- flags_old = flagger_old.flags[var]
+ if flagger_old is not None and var in flagger_old.columns:
+ flags_old = flagger_old[var]
ref_plotdict = _getPlotdict(data_old, flags_old, flagger_old, var)
# check flags-index changes:
@@ -428,12 +428,12 @@ def _getDataFromVar(
# calculate old-flags and update flags, like BADs,
# to show only freshly new set values
- unflagged = plotdict["unflagged"]
+ unflagged = plotdict.get("unflagged", pd.Series(dtype=float))
diff = unchanged.index.difference(unflagged.index)
plotdict["old-flags"] = unchanged.loc[diff]
for field in ["bad", "suspicious", "good"]:
- data = plotdict[field]
- isect = changed.index & data.index
+ data = plotdict.get(field, pd.Series(dtype=float))
+ isect = changed.index.intersection(data.index)
plotdict[field] = data.loc[isect]
return plotdict, ref_plotdict
@@ -540,7 +540,7 @@ def _splitOldAndNew(old: pd.Series, new: pd.Series):
of locations seen from new. This means, the rest marks locations, that
are present(!) in new, but its data differs from old.
"""
- idx = old.index & new.index
+ idx = old.index.intersection(new.index)
both_nan = old.loc[idx].isna() & new.loc[idx].isna()
mask = (new.loc[idx] == old[idx]) | both_nan
old_idx = mask[mask].index
@@ -553,12 +553,10 @@ def _splitByFlag(flags: pd.Series, flagger, var: str):
Splits flags in the five distinct bins: GOOD, SUSPICIOUS, BAD, UNFLAGGED and NaNs.
"""
n = flags.isna()
- loc = flags.dropna().index
- g = flagger.isFlagged(field=var, loc=loc, flag=GOOD, comparator="==")
- b = flagger.isFlagged(field=var, loc=loc, flag=BAD, comparator="==")
- u = flagger.isFlagged(field=var, loc=loc, flag=UNFLAGGED, comparator="==")
- s = flagger.isFlagged(field=var, loc=loc, flag=BAD, comparator="<")
- s = flagger.isFlagged(field=var, loc=loc, flag=GOOD, comparator=">") & s
+ b = flags >= BAD
+ g = flags < UNFLAGGED
+ u = flags == UNFLAGGED
+ s = (flags > UNFLAGGED) & (flags < BAD)
return g[g], s[s], b[b], u[u], n[n]
--
GitLab
From 8a7c697a8446c4da6874649abf3dc79748499e29 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 5 Mar 2021 14:56:54 +0100
Subject: [PATCH 034/180] fixed BAD/DOUBTFUL/GOOD; str-flag now float in
defaults and docstrings; UNFLAGGED now is -inf
---
saqc/common.py | 4 ++--
saqc/core/modules/flagtools.py | 7 ++++---
saqc/core/modules/interpolation.py | 7 ++++---
saqc/core/visitor.py | 1 +
saqc/funcs/changepoints.py | 2 +-
saqc/funcs/drift.py | 2 +-
saqc/funcs/flagtools.py | 16 +++++++--------
saqc/funcs/generic.py | 7 ++++---
saqc/funcs/interpolation.py | 6 ++----
saqc/funcs/resampling.py | 21 +++++++-------------
test/core/test_masking.py | 32 ++++++++++++++++--------------
test/funcs/test_proc_functions.py | 5 +++--
12 files changed, 53 insertions(+), 57 deletions(-)
diff --git a/saqc/common.py b/saqc/common.py
index 013797048..89db985ec 100644
--- a/saqc/common.py
+++ b/saqc/common.py
@@ -12,12 +12,12 @@ __all__ = [
import numpy as np
UNTOUCHED = np.nan
-UNFLAGGED = 0.
+UNFLAGGED = -np.inf
+GOOD = 0
DOUBTFUL = 25.
BAD = 255.
# aliases
-GOOD = UNFLAGGED
DOUBT = DOUBTFUL
diff --git a/saqc/core/modules/flagtools.py b/saqc/core/modules/flagtools.py
index 385957aee..c9c1b0892 100644
--- a/saqc/core/modules/flagtools.py
+++ b/saqc/core/modules/flagtools.py
@@ -8,6 +8,7 @@ import pandas as pd
from dios.dios import DictOfSeries
from saqc.core.modules.base import ModuleBase
+from saqc.common import *
class FlagTools(ModuleBase):
@@ -15,7 +16,7 @@ class FlagTools(ModuleBase):
def clearFlags(self, field: str, **kwargs):
return self.defer("clearFlags", locals())
- def forceFlags(self, field: str, flag: Any, **kwargs):
+ def forceFlags(self, field: str, flag: float = BAD, **kwargs):
return self.defer("forceFlags", locals())
def flagDummy(self, field: str, **kwargs):
@@ -24,10 +25,10 @@ class FlagTools(ModuleBase):
def flagForceFail(self, field: str, **kwargs):
return self.defer("flagForceFail", locals())
- def flagUnflagged(self, field: str, flag: Optional[Any] = None, **kwargs):
+ def flagUnflagged(self, field: str, flag: float = BAD, **kwargs):
return self.defer("flagUnflagged", locals())
- def flagGood(self, field: str, flag: Optional[Any] = None, **kwargs):
+ def flagGood(self, field: str, flag: float = BAD, **kwargs):
return self.defer("flagGood", locals())
def flagManual(
diff --git a/saqc/core/modules/interpolation.py b/saqc/core/modules/interpolation.py
index d0675883d..c73da2563 100644
--- a/saqc/core/modules/interpolation.py
+++ b/saqc/core/modules/interpolation.py
@@ -7,6 +7,7 @@ from typing_extensions import Literal
import numpy as np
import pandas as pd
+from saqc.common import *
from saqc.core.modules.base import ModuleBase
@@ -30,9 +31,9 @@ class Interpolation(ModuleBase):
method: Literal["linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric", "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"],
inter_order: int = 2,
inter_limit: int = 2,
- interpol_flag: Any = "UNFLAGGED",
+ interpol_flag: float = UNFLAGGED,
downgrade_interpolation: bool = False,
- not_interpol_flags: Optional[Union[Any, Sequence[Any]]] = None,
+ not_interpol_flags: Optional[Union[float, Sequence[float]]] = None,
**kwargs
):
return self.defer("interpolateInvalid", locals())
@@ -59,7 +60,7 @@ class Interpolation(ModuleBase):
method: Literal["linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric", "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"],
inter_order: int = 2,
inter_limit: int = 2,
- interpol_flag: Any = "UNFLAGGED",
+ interpol_flag: float = UNFLAGGED,
downgrade_interpolation: bool = False,
not_interpol_flags: Optional[Union[Any, Sequence[Any]]] = None,
**kwargs
diff --git a/saqc/core/visitor.py b/saqc/core/visitor.py
index b9466c34c..cf04e7605 100644
--- a/saqc/core/visitor.py
+++ b/saqc/core/visitor.py
@@ -40,6 +40,7 @@ ENVIRONMENT = {
"zLog": ts_ops.zeroLog,
}
+# todo: how does a user pass flags now
RESERVED = {"GOOD", "BAD", "UNFLAGGED", "NODATA"}
diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py
index f11b413ba..0e17bc540 100644
--- a/saqc/funcs/changepoints.py
+++ b/saqc/funcs/changepoints.py
@@ -163,7 +163,7 @@ def assignChangePointCluster(
reduction window. Second input parameter holds the result from the thresh_func evaluation.
The default reduction function just selects the value that maximizes the stat_func.
flag_changepoints : bool, default False
- If true, the points, where there is a change in data modelling regime detected get flagged bad.
+ If true, the points, where there is a change in data modelling regime detected get flagged BAD.
model_by_resids : bool, default False
If True, the data is replaced by the stat_funcs results instead of regime labels.
assign_cluster : bool, default True
diff --git a/saqc/funcs/drift.py b/saqc/funcs/drift.py
index dfb530b04..dc2265e8a 100644
--- a/saqc/funcs/drift.py
+++ b/saqc/funcs/drift.py
@@ -394,7 +394,7 @@ def correctExponentialDrift(
The number of values the mean is computed over, for obtaining the value level directly after and
directly before maintenance event. This values are needed for shift calibration. (see above description)
flag_maint_period : bool, default False
- Wheather or not to flag BAD the values directly obtained while maintenance.
+ Whether or not to flag the values obtained while maintenance.
Returns
-------
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index 99337a70b..fb9522324 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -16,7 +16,7 @@ import warnings
@register(masking='field', module="flagtools")
def forceFlags(
- data: DictOfSeries, field: ColumnName, flagger: Flagger, flag: float, **kwargs
+ data: DictOfSeries, field: ColumnName, flagger: Flagger, flag: float = BAD, **kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
Set whole column to a flag value.
@@ -29,7 +29,7 @@ def forceFlags(
columns name that holds the data
flagger : Flagger
flagger object
- flag : float
+ flag : float, default BAD
flag to set
kwargs : dict
unused
@@ -80,12 +80,10 @@ def clearFlags(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs
@register(masking='field', module="flagtools")
def flagUnflagged(
- data: DictOfSeries, field: ColumnName, flagger: Flagger, flag: float, **kwargs
+ data: DictOfSeries, field: ColumnName, flagger: Flagger, flag: float = BAD, **kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
- Function sets the GOOD flag to all values flagged better then GOOD.
- If there is an entry 'flag' in the kwargs dictionary passed, the
- function sets the kwargs['flag'] flag to all values flagged better kwargs['flag']
+ Function sets a flag at all unflagged positions.
Parameters
----------
@@ -95,7 +93,7 @@ def flagUnflagged(
The fieldname of the column, holding the data-to-be-flagged.
flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional informations related to `data`.
- flag : float
+ flag : float, default BAD
flag value to set, has NO default
kwargs : Dict
unused
@@ -118,9 +116,9 @@ def flagUnflagged(
@register(masking='field', module="flagtools")
-def flagGood(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+def flagGood(data: DictOfSeries, field: ColumnName, flagger: Flagger, flag=BAD, **kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
- Function sets the GOOD flag to all values flagged better then GOOD.
+ Function sets the GOOD flag at all unflagged positions.
Parameters
----------
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index 8753ca154..e65e4f354 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -21,7 +21,7 @@ _OP = {'<': op.lt, '<=': op.le, '==': op.eq, '!=': op.ne, '>': op.gt, '>=': op.g
def _dslIsFlagged(
- flagger: Flagger, var: pd.Series, flag: Any = None, comparator: str = ">="
+ flagger: Flagger, var: pd.Series, flag: float = UNFLAGGED, comparator: str = ">"
) -> Union[pd.Series, DictOfSeries]:
"""
helper function for `flag`
@@ -187,10 +187,11 @@ def flag(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd.Se
>>> lambda level: isflagged(level)
You can furthermore specify a flagging level, you want to compare the flags to. For example, for flagging
- 'temperature', if 'level' is flagged at a level named 'doubtfull' or worse, use:
+ 'temperature', if 'level' is flagged at a level named DOUBTFUL or worse, use:
- >>> lambda level: isflagged(level, flag='doubtfull', comparator='<=')
+ >>> lambda level: isflagged(level, flag=DOUBTFUL, comparator='>')
+ # TODO : fix text
If you are unsure about the used flaggers flagging level names, you can use the reserved key words BAD, UNFLAGGED
and GOOD, to refer to the worst (BAD), best(GOOD) or unflagged (UNFLAGGED) flagging levels. For example.
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index 3a5d73cb2..425335ef4 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -52,9 +52,8 @@ def interpolateByRolling(
min_periods : int
Minimum number of valid (not np.nan) values that have to be available in a window for its aggregation to be
computed.
- flag : float, default 0
+ flag : float, default UNFLAGGED
Flag that is to be inserted for the interpolated values. If ``None`` no flags are set.
- Defaults to ``0`` aka. ``UNFLAGGED``.
Returns
-------
@@ -127,9 +126,8 @@ def interpolateInvalid(
order.
inter_limit : int, default 2
Maximum number of consecutive 'nan' values allowed for a gap to be interpolated.
- flag : float or None, default 0
+ flag : float or None, default UNFLAGGED
Flag that is to be inserted for the interpolated values. If ``None`` no flags are set.
- Defaults to ``0`` aka. ``UNFLAGGED``.
downgrade_interpolation : bool, default False
If interpolation can not be performed at `inter_order`, because not enough values are present or the order
is not implemented for the passed method, automatically try to interpolate at ``inter_order-1``.
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index d6fed78ee..63155c27e 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -363,7 +363,7 @@ def _shift(
freq: str,
method: Literal["fshift", "bshift", "nshift"]="nshift",
to_drop: Optional[Union[Any, Sequence[Any]]]=None,
- empty_intervals_flag: Optional[str]=None,
+ empty_intervals_flag: float = UNFLAGGED,
freq_check: Optional[Literal["check", "auto"]]=None,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
@@ -393,9 +393,8 @@ def _shift(
method: {'fshift', 'bshift', 'nshift'}, default 'nshift'
Specifies if datapoints get propagated forwards, backwards or to the nearest grid timestamp. See function
description for more details.
- empty_intervals_flag : {None, str}, default None
- A Flag, that you want to assign to grid points, where no values are avaible to be shifted to.
- Default triggers UNFLAGGED to be assigned.
+ empty_intervals_flag : float, default UNFLAGGED
+ The Flag, that is assigned to grid points, if no values are available to be shifted to.
to_drop : {None, str, List[str]}, default None
Flags that refer to values you want to drop before shifting - effectively, excluding values that are flagged
with a flag in to_drop from the shifting process. Default - to_drop = None - results in BAD
@@ -420,9 +419,6 @@ def _shift(
datcol = data[field]
flagscol = flagger[field]
- if empty_intervals_flag is None:
- empty_intervals_flag = UNFLAGGED
-
drop_mask = getDropMask(field, to_drop, flagger, BAD)
drop_mask |= datcol.isna()
datcol[drop_mask] = np.nan
@@ -470,7 +466,7 @@ def resample(
max_invalid_consec_f: Optional[int]=None,
max_invalid_total_f: Optional[int]=None,
flag_agg_func: Callable[[pd.Series], float]=max,
- empty_intervals_flag: Optional[Any]=None,
+ empty_intervals_flag: float = BAD,
to_drop: Optional[Union[Any, Sequence[Any]]]=None,
all_na_2_empty: bool=False,
freq_check: Optional[Literal["check", "auto"]]=None,
@@ -528,15 +524,14 @@ def resample(
Also this is the flag assigned to invalid/empty intervals.
max_invalid_consec_f : {None, int}, default None
Same as `max_invalid_total_f`, only applying onto flags. The flag regarded as "invalid" value, is the one passed
- to empty_intervals_flag (default=BAD). Also this is the flag assigned to invalid/empty intervals.
+ to empty_intervals_flag. Also this is the flag assigned to invalid/empty intervals.
flag_agg_func : Callable, default: max
The function you want to aggregate the flags with. It should be capable of operating on the flags dtype
(usually ordered categorical).
- empty_intervals_flag : {None, str}, default None
+ empty_intervals_flag : float, default BAD
A Flag, that you want to assign to invalid intervals. Invalid are those intervals, that contain nan values only,
or no values at all. Furthermore the empty_intervals_flag is the flag, serving as "invalid" identifyer when
- checking for `max_total_invalid_f` and `max_consec_invalid_f patterns`. Default triggers ``BAD`` to be
- assigned.
+ checking for `max_total_invalid_f` and `max_consec_invalid_f patterns`.
to_drop : {None, str, List[str]}, default None
Flags that refer to values you want to drop before resampling - effectively excluding values that are flagged
with a flag in to_drop from the resampling process - this means that they also will not be counted in the
@@ -561,8 +556,6 @@ def resample(
data = data.copy()
datcol = data[field]
flagscol = flagger[field]
- if empty_intervals_flag is None:
- empty_intervals_flag = BAD
drop_mask = getDropMask(field, to_drop, flagger, [])
datcol.drop(datcol[drop_mask].index, inplace=True)
diff --git a/test/core/test_masking.py b/test/core/test_masking.py
index 48207621e..6236a55e2 100644
--- a/test/core/test_masking.py
+++ b/test/core/test_masking.py
@@ -13,6 +13,7 @@ from hypothesis.strategies import (
)
from saqc.common import *
+from saqc.flagger import Flagger, initFlagsLike
from saqc.core.register import _maskData, _unmaskData
from test.common import dataFieldFlagger, MAX_EXAMPLES
@@ -27,9 +28,10 @@ def test_maskingMasksData(data_field_flagger):
"""
test if flagged values are replaced by np.nan
"""
+ flagger: Flagger
data_in, field, flagger = data_field_flagger
- data_masked, _ = _maskData(data_in, flagger, columns=[field], to_mask=flagger.BAD)
- assert data_masked.aloc[flagger.isFlagged(flagger.BAD)].isna().all(axis=None)
+ data_masked, _ = _maskData(data_in, flagger, columns=[field], to_mask=BAD)
+ assert data_masked.aloc[flagger.toDios() == BAD].isna().all(axis=None)
@settings(max_examples=MAX_EXAMPLES, deadline=None)
@@ -42,9 +44,9 @@ def test_dataMutationPreventsUnmasking(data_field_flagger):
filler = -9999
data_in, field, flagger = data_field_flagger
- data_masked, mask = _maskData(data_in, flagger, columns=[field], to_mask=flagger.BAD)
+ data_masked, mask = _maskData(data_in, flagger, columns=[field], to_mask=BAD)
data_masked[field] = filler
- data_out = _unmaskData(data_in, mask, data_masked, flagger, to_mask=flagger.BAD)
+ data_out = _unmaskData(data_in, mask, data_masked, flagger, to_mask=BAD)
assert (data_out[field] == filler).all(axis=None)
@@ -56,10 +58,10 @@ def test_flaggerMutationPreventsUnmasking(data_field_flagger):
if `flagger` is mutated after `_maskData`, `_unmaskData` should be a no-op
"""
data_in, field, flagger = data_field_flagger
- data_masked, mask = _maskData(data_in, flagger, columns=[field], to_mask=flagger.BAD)
- flagger = flagger.setFlags(field, flag=flagger.UNFLAGGED, force=True)
- data_out = _unmaskData(data_in, mask, data_masked, flagger, to_mask=flagger.BAD)
- assert (data_out.loc[flagger.isFlagged(field, flag=flagger.BAD), field].isna()).all(axis=None)
+ data_masked, mask = _maskData(data_in, flagger, columns=[field], to_mask=BAD)
+ flagger = flagger[field] = UNFLAGGED
+ data_out = _unmaskData(data_in, mask, data_masked, flagger, to_mask=BAD)
+ assert (data_out.loc[flagger[field] == BAD, field].isna()).all(axis=None)
@settings(max_examples=MAX_EXAMPLES, deadline=None)
@@ -74,17 +76,17 @@ def test_reshapingPreventsUnmasking(data_field_flagger):
filler = -1111
data_in, field, flagger = data_field_flagger
- data_masked, mask = _maskData(data_in, flagger, columns=[field], to_mask=flagger.BAD)
+ data_masked, mask = _maskData(data_in, flagger, columns=[field], to_mask=BAD)
# mutate indexes of `data` and `flagger`
index = data_masked[field].index.to_series()
index.iloc[-len(data_masked[field])//2:] += pd.Timedelta("7.5Min")
data_masked[field] = pd.Series(data=filler, index=index)
- flags = flagger.getFlags()
- flags[field] = pd.Series(data=flags[field].values, index=index)
- flagger = flagger.initFlags(flags=flags)
- data_out = _unmaskData(data_in, mask, data_masked, flagger, to_mask=flagger.BAD)
+ flagger.drop(field)
+ flagger[field] = pd.Series(data=flagger[field].values, index=index)
+
+ data_out = _unmaskData(data_in, mask, data_masked, flagger, to_mask=BAD)
assert (data_out[field] == filler).all(axis=None)
@@ -95,8 +97,8 @@ def test_unmaskingInvertsMasking(data_field_flagger):
unmasking data should invert the masking
"""
data_in, field, flagger = data_field_flagger
- data_masked, mask = _maskData(data_in, flagger, columns=[field], to_mask=flagger.BAD)
- data_out = _unmaskData(data_in, mask, data_masked, flagger, to_mask=flagger.BAD)
+ data_masked, mask = _maskData(data_in, flagger, columns=[field], to_mask=BAD)
+ data_out = _unmaskData(data_in, mask, data_masked, flagger, to_mask=BAD)
assert data_in.to_df().equals(data_out.to_df())
diff --git a/test/funcs/test_proc_functions.py b/test/funcs/test_proc_functions.py
index 3aa5c2c1b..a7c99fc77 100644
--- a/test/funcs/test_proc_functions.py
+++ b/test/funcs/test_proc_functions.py
@@ -16,6 +16,7 @@ from saqc.funcs.drift import correctOffset
from saqc.funcs.interpolation import interpolateByRolling, interpolateInvalid, interpolateIndex
from saqc.funcs.resampling import resample
from saqc.lib.ts_operators import linearInterpolation, polynomialInterpolation
+from saqc.common import *
from test.common import TESTFLAGGER
@@ -27,13 +28,13 @@ def test_rollingInterpolateMissing(course_5, flagger):
data = dios.DictOfSeries(data)
flagger = flagger.initFlags(data)
dataInt, *_ = interpolateByRolling(
- data, field, flagger, 3, func=np.median, center=True, min_periods=0, interpol_flag="UNFLAGGED"
+ data, field, flagger, 3, func=np.median, center=True, min_periods=0, interpol_flag=UNFLAGGED
)
# import pdb
# pdb.set_trace()
assert dataInt[field][characteristics["missing"]].notna().all()
dataInt, *_ = interpolateByRolling(
- data, field, flagger, 3, func=np.nanmean, center=False, min_periods=3, interpol_flag="UNFLAGGED"
+ data, field, flagger, 3, func=np.nanmean, center=False, min_periods=3, interpol_flag=UNFLAGGED
)
assert dataInt[field][characteristics["missing"]].isna().all()
--
GitLab
From 4d8d90af0d240ec8691558aeaf00b7241c280a9c Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 5 Mar 2021 15:02:36 +0100
Subject: [PATCH 035/180] fixed todo -> TODO
---
saqc/core/core.py | 10 ++++------
saqc/core/lib.py | 2 +-
saqc/core/visitor.py | 2 +-
saqc/funcs/changepoints.py | 6 +++---
saqc/funcs/curvefit.py | 4 ++--
saqc/funcs/generic.py | 2 +-
saqc/funcs/interpolation.py | 4 ++--
saqc/funcs/outliers.py | 4 ++--
saqc/funcs/resampling.py | 2 +-
saqc/funcs/rolling.py | 3 ++-
saqc/funcs/scores.py | 2 +-
saqc/lib/tools.py | 2 +-
test/funcs/test_functions.py | 2 +-
13 files changed, 22 insertions(+), 23 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index de726326d..3fdd00a5a 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -2,11 +2,9 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
-"""
-TODOS:
- - integrate plotting into the api
- - `data` and `flagger` as arguments to `getResult`
-"""
+# TODO:
+# - integrate plotting into the api
+# - `data` and `flagger` as arguments to `getResult`
import logging
import copy as stdcopy
@@ -49,7 +47,7 @@ def _handleErrors(exc: Exception, field: str, control: APIController, func: SaQC
raise exc
-# todo: shouldt the code/function go to Saqc.__init__ ?
+# TODO: shouldt the code/function go to Saqc.__init__ ?
def _prepInput(data, flags):
dios_like = (dios.DictOfSeries, pd.DataFrame)
diff --git a/saqc/core/lib.py b/saqc/core/lib.py
index 7ee39f768..2236e3b63 100644
--- a/saqc/core/lib.py
+++ b/saqc/core/lib.py
@@ -14,7 +14,7 @@ class ColumnSelector:
regex: bool
-# todo: this seems obsolete
+# TODO: this seems obsolete
@dataclass
class APIController:
plot: bool
diff --git a/saqc/core/visitor.py b/saqc/core/visitor.py
index cf04e7605..560d87748 100644
--- a/saqc/core/visitor.py
+++ b/saqc/core/visitor.py
@@ -40,7 +40,7 @@ ENVIRONMENT = {
"zLog": ts_ops.zeroLog,
}
-# todo: how does a user pass flags now
+# TODO: how does a user pass flags now
RESERVED = {"GOOD", "BAD", "UNFLAGGED", "NODATA"}
diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py
index 0e17bc540..00d201f77 100644
--- a/saqc/funcs/changepoints.py
+++ b/saqc/funcs/changepoints.py
@@ -30,7 +30,7 @@ def flagChangePoints(
fwd_window: Optional[FreqString]=None,
min_periods_fwd: Optional[IntegerWindow]=None,
closed: Literal["right", "left", "both", "neither"]="both",
- try_to_jit: bool=True, # todo rm, not a user decision
+ try_to_jit: bool=True, # TODO rm, not a user decision
reduce_window: FreqString=None,
reduce_func: Callable[[np.ndarray, np.ndarray], int]=lambda x, _: x.argmax(),
**kwargs
@@ -107,7 +107,7 @@ def assignChangePointCluster(
fwd_window: str=None,
min_periods_fwd: Optional[int]=None,
closed: Literal["right", "left", "both", "neither"]="both",
- try_to_jit: bool=True, # todo: rm, not a user decision
+ try_to_jit: bool=True, # TODO: rm, not a user decision
reduce_window: str=None,
reduce_func: Callable[[np.ndarray, np.ndarray], float]=lambda x, _: x.argmax(),
model_by_resids: bool=False,
@@ -242,7 +242,7 @@ def assignChangePointCluster(
flagger[:, field] = UNFLAGGED
if flag_changepoints:
- # todo: does not respect kwargs[flag]
+ # TODO: does not respect kwargs[flag]
flagger[det_index, field] = BAD
return data, flagger
diff --git a/saqc/funcs/curvefit.py b/saqc/funcs/curvefit.py
index 3a98dfdde..9623d49d1 100644
--- a/saqc/funcs/curvefit.py
+++ b/saqc/funcs/curvefit.py
@@ -102,7 +102,7 @@ def fitPolynomial(data: DictOfSeries, field: str, flagger: Flagger,
Flags values may have changed relatively to the flagger input.
"""
- # todo: some (rater large) parts are functional similar to saqc.funcs.rolling.roll
+ # TODO: some (rater large) parts are functional similar to saqc.funcs.rolling.roll
if data[field].empty:
return data, flagger
data = data.copy()
@@ -195,7 +195,7 @@ def fitPolynomial(data: DictOfSeries, field: str, flagger: Flagger,
if eval_flags:
# with the new flagger we dont have to care
# about to set NaNs to the original flags anymore
- # todo: we does not get any flags here, because of masking=field
+ # TODO: we does not get any flags here, because of masking=field
worst = flagger[field].rolling(winsz, center=True, min_periods=min_periods).max()
flagger[field] = worst
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index e65e4f354..ed64f7ca0 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -115,7 +115,7 @@ def process(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd
"""
data[field] = _execGeneric(flagger, data, func, field, nodata).squeeze()
- # todo: the former comment wished to overwrite the column, but i'm not sure -- palmb
+ # TODO: the former comment wished to overwrite the column, but i'm not sure -- palmb
if field in flagger:
flagger.drop(field)
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index 425335ef4..a615aee92 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -155,7 +155,7 @@ def interpolateInvalid(
)
interpolated = data[field].isna() & inter_data.notna()
- # todo: remove with version 2.0
+ # TODO: remove with version 2.0
if not_interpol_flags is not None:
raise ValueError("'not_interpol_flags' is deprecated")
@@ -179,7 +179,7 @@ def interpolateIndex(
empty_intervals_flag: Any=None,
grid_field: str=None,
inter_limit: int=2,
- freq_check: Optional[Literal["check", "auto"]]=None, # todo: rm not a user decision
+ freq_check: Optional[Literal["check", "auto"]]=None, # TODO: rm not a user decision
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index bc7f5e06c..818a80ed0 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -513,7 +513,7 @@ def flagRaise(
mean_raise_factor: float=2.,
min_slope: Optional[float]=None,
min_slope_weight: float=0.8,
- numba_boost: bool=True, # todo: rm, not a user decision
+ numba_boost: bool=True, # TODO: rm, not a user decision
**kwargs,
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -734,7 +734,7 @@ def flagOffset(
tolerance: float,
window: Union[IntegerWindow, FreqString],
rel_thresh: Optional[float]=None,
- numba_kickin: int=200000, # todo: rm, not a user decision
+ numba_kickin: int=200000, # TODO: rm, not a user decision
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 63155c27e..3a4d8ce78 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -344,7 +344,7 @@ def shift(
method: Literal["fshift", "bshift", "nshift"]="nshift",
to_drop: Optional[Union[Any, Sequence[Any]]]=None,
empty_intervals_flag: Optional[str]=None,
- freq_check: Optional[Literal["check", "auto"]]=None, # todo: not a user decision
+ freq_check: Optional[Literal["check", "auto"]]=None, # TODO: not a user decision
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
diff --git a/saqc/funcs/rolling.py b/saqc/funcs/rolling.py
index 99f6be681..a0740e511 100644
--- a/saqc/funcs/rolling.py
+++ b/saqc/funcs/rolling.py
@@ -124,7 +124,8 @@ def roll(
if eval_flags:
# with the new flagger we dont have to care
# about to set NaNs to the original flags anymore
- # todo: we does not get any flags here, because of masking=field
+
+ # TODO: we does not get any flags here, because of masking=field
worst = flagger[field].rolling(winsz, center=True, min_periods=min_periods).max()
flagger[field] = worst
diff --git a/saqc/funcs/scores.py b/saqc/funcs/scores.py
index 1c670b287..bdc27b597 100644
--- a/saqc/funcs/scores.py
+++ b/saqc/funcs/scores.py
@@ -156,7 +156,7 @@ def assignKNNScore(
score_ser[partition.index] = resids
- # todo: this unconditionally overwrite a column, may we should fire a warning ? -- palmb
+ # TODO: this unconditionally overwrite a column, may we should fire a warning ? -- palmb
if target_field in flagger.columns:
flagger.drop(target_field)
flagger[target_field] = pd.Series(UNFLAGGED, index=score_ser.index, dtype=float)
diff --git a/saqc/lib/tools.py b/saqc/lib/tools.py
index b70f660dd..dec366b98 100644
--- a/saqc/lib/tools.py
+++ b/saqc/lib/tools.py
@@ -308,7 +308,7 @@ def isQuoted(string):
return bool(re.search(r"'.*'|\".*\"", string))
-# todo: GL167
+# TODO: GL167
def getDropMask(field, to_drop, flagger, default):
drop_mask = pd.Series(False, index=flagger[field].index)
if to_drop is None:
diff --git a/test/funcs/test_functions.py b/test/funcs/test_functions.py
index 2a466df14..1a56f8e8a 100644
--- a/test/funcs/test_functions.py
+++ b/test/funcs/test_functions.py
@@ -84,7 +84,7 @@ def test_forceFlags(data, field):
assert all(flagger[field] == DOUBT)
-# todo: @luenensc: i dont get the test -- palmb
+# TODO: @luenensc: i dont get the test -- palmb
def test_flagIsolated(data, field):
flagger = initFlagsLike(data)
--
GitLab
From 5a0b99fa5d38a291ff12f5cf34db2556d4da22d0 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 5 Mar 2021 15:03:50 +0100
Subject: [PATCH 036/180] [BUGFIX] untouched is not unflagged !
---
saqc/flagger/history.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/saqc/flagger/history.py b/saqc/flagger/history.py
index 65759b25c..551dba6c4 100644
--- a/saqc/flagger/history.py
+++ b/saqc/flagger/history.py
@@ -160,7 +160,7 @@ class History:
self.mask[pos] = pd.Series(True, index=s.index, dtype=bool)
if force:
- touched = np.isfinite(s)
+ touched = s.notna()
self.mask.iloc[touched, :pos] = False
self.hist[pos] = s
@@ -287,7 +287,7 @@ class History:
----------
index : pd.Index
the index to reindex to.
- fill_value_last : float, default 0
+ fill_value_last : float, default UNFLAGGED
value to fill nan's (UNTOUCHED) in the last column. Defaults to 0 (UNFLAGGED).
Returns
--
GitLab
From fba49a89f4a8b5ba684c4ba63ff8bc2455083139 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Wed, 10 Mar 2021 11:22:38 +0100
Subject: [PATCH 037/180] added a selection of fuzzy function tests
---
test/funcs/test_functions_fuzzy.py | 97 ++++++++++++++++++++++++++++++
1 file changed, 97 insertions(+)
create mode 100644 test/funcs/test_functions_fuzzy.py
diff --git a/test/funcs/test_functions_fuzzy.py b/test/funcs/test_functions_fuzzy.py
new file mode 100644
index 000000000..905a28611
--- /dev/null
+++ b/test/funcs/test_functions_fuzzy.py
@@ -0,0 +1,97 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from saqc.core.register import FUNC_MAP
+
+from hypothesis import given, settings
+from hypothesis.strategies import (
+ data,
+)
+
+from test.common import MAX_EXAMPLES, functionKwargs
+
+
+@settings(max_examples=MAX_EXAMPLES, deadline=None)
+@given(drawer=data())
+def callWontBreak(drawer, func_name: str):
+ func = FUNC_MAP[func_name]
+ kwargs = drawer.draw(functionKwargs(func))
+ func(**kwargs)
+
+
+# breaks
+# ------
+
+# NOTE:
+# needs a more alaborated test, as it calls into
+# `changepoints.assignChangePointClusters`
+# def test_breaks_flagJumps():
+# callWontBreak("breaks.flagJumps")
+# def test_breaks_flagIsolated():
+# callWontBreak("breaks.flagIsolated")
+
+def test_breaks_flagMissing():
+ callWontBreak("breaks.flagMissing")
+
+
+# constants
+# ---------
+
+def test_constats_flagConstats():
+ callWontBreak("constants.flagConstants")
+
+def test_constants_flagByVariance():
+ callWontBreak("constants.flagByVariance")
+
+
+# flagtools
+# ---------
+
+def test_flagtools_clearFlags():
+ callWontBreak("flagtools.clearFlags")
+
+def test_flagtools_forceFlags():
+ callWontBreak("flagtools.clearFlags")
+
+# NOTE:
+# all of the following tests fail to sample data for `flag=typing.Any`
+# with the new flagger in place this should be easy to fix
+# def test_flagtools_flagGood():
+# callWontBreak("flagtools.flagGood")
+
+# def test_flagtools_flagUnflagged():
+# callWontBreak("flagtools.flagUnflagged")
+
+# def test_flagtools_flagManual():
+# callWontBreak("flagtools.flagManual")
+
+
+# outliers
+# --------
+
+# NOTE: needs a more elaborated test, I guess
+# def test_outliers_flagByStray():
+# callWontBreak("outliers.flagByStray")
+
+# NOTE: fails in a strategy, maybe `Sequence[ColumnName]`
+# def test_outliers_flagMVScores():
+# callWontBreak("outliers.flagMVScores")
+
+# NOTE:
+# fails as certain combinations of frquency strings don't make sense
+# a more elaborate test is needed
+# def test_outliers_flagRaise():
+# callWontBreak("outliers.flagRaise")
+
+def test_outliers_flagMAD():
+ callWontBreak("outliers.flagMAD")
+
+def test_outliers_flagByGrubbs():
+ callWontBreak("outliers.flagByGrubbs")
+
+def test_outliers_flagRange():
+ callWontBreak("outliers.flagRange")
+
+# NOTE: fails in a strategy, maybe `Sequence[ColumnName]`
+# def test_outliers_flagCrossStatistic():
+# callWontBreak("outliers.flagCrossStatistic")
--
GitLab
From ae00e0e282277ef3eee83c5298fd2b3e3ba03915 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 12 Mar 2021 12:23:36 +0100
Subject: [PATCH 038/180] moved fuzzy tests to own dir
---
test/common.py | 159 ----------------
testsfuzzy/__init__.py | 1 +
testsfuzzy/init.py | 169 ++++++++++++++++++
.../test_functions.py | 60 ++++---
{test/core => testsfuzzy}/test_masking.py | 2 +-
5 files changed, 207 insertions(+), 184 deletions(-)
create mode 100644 testsfuzzy/__init__.py
create mode 100644 testsfuzzy/init.py
rename test/funcs/test_functions_fuzzy.py => testsfuzzy/test_functions.py (63%)
rename {test/core => testsfuzzy}/test_masking.py (98%)
diff --git a/test/common.py b/test/common.py
index f774cd5ed..f61ddac3e 100644
--- a/test/common.py
+++ b/test/common.py
@@ -2,32 +2,11 @@
# -*- coding: utf-8 -*-
import io
-from typing import get_type_hints
-
import numpy as np
import pandas as pd
import dios
-from hypothesis.strategies import (
- lists,
- sampled_from,
- composite,
- from_regex,
- sampled_from,
- datetimes,
- integers,
- register_type_strategy,
- from_type,
-)
-from hypothesis.extra.numpy import arrays, from_dtype
-from hypothesis.strategies._internal.types import _global_type_lookup
-
-from dios import DictOfSeries
-
from saqc.common import *
-from saqc.core.register import FUNC_MAP
-from saqc.core.lib import SaQCFunction
-from saqc.lib.types import FreqString, ColumnName, IntegerWindow
from saqc.flagger import Flagger, initFlagsLike
@@ -63,141 +42,3 @@ def writeIO(content):
return f
-MAX_EXAMPLES = 50 #100000
-
-
-@composite
-def dioses(draw, min_cols=1):
- """
- initialize data according to the current restrictions
- """
- # NOTE:
- # The following restriction showed up and should be enforced during init:
- # - Column names need to satisify the following regex: [A-Za-z0-9_-]+
- # - DatetimeIndex needs to be sorted
- # - Integer values larger than 2**53 lead to numerical instabilities during
- # the integer->float->integer type conversion in _maskData/_unmaskData.
-
- cols = draw(lists(columnNames(), unique=True, min_size=min_cols))
- columns = {
- c: draw(dataSeries(min_size=3))
- for c in cols
- }
- return DictOfSeries(columns)
-
-import numbers
-
-@composite
-def dataSeries(draw, min_size=0, max_size=100, dtypes=("float32", "float64", "int32", "int64")):
- if np.isscalar(dtypes):
- dtypes = (dtypes,)
-
- dtype = np.dtype(draw(sampled_from(dtypes)))
- if issubclass(dtype.type, numbers.Integral):
- info = np.iinfo(dtype)
- elif issubclass(dtype.type, numbers.Real):
- info = np.finfo(dtype)
- else:
- raise ValueError("only numerical dtypes are supported")
- # we don't want to fail just because of overflows
- elements = from_dtype(dtype, min_value=info.min+1, max_value=info.max-1)
-
- index = draw(daterangeIndexes(min_size=min_size, max_size=max_size))
- values = draw(arrays(dtype=dtype, elements=elements, shape=len(index)))
- return pd.Series(data=values, index=index)
-
-
-@composite
-def columnNames(draw):
- return draw(from_regex(r"[A-Za-z0-9_-]+", fullmatch=True))
-
-
-@composite
-def flaggers(draw, data):
- """
- initialize a flagger and set some flags
- """
- # flagger = draw(sampled_from(TESTFLAGGER)).initFlags(data)
- flagger = initFlagsLike(data)
- for col, srs in data.items():
- loc_st = lists(sampled_from(sorted(srs.index)), unique=True, max_size=len(srs)-1)
- flagger[draw(loc_st), col] = BAD
- return flagger
-
-
-@composite
-def functions(draw, module: str=None):
- samples = tuple(FUNC_MAP.values())
- if module:
- samples = tuple(f for f in samples if f.name.startswith(module))
- # samples = [FUNC_MAP["drift.correctExponentialDrift"]]
- return draw(sampled_from(samples))
-
-
-@composite
-def daterangeIndexes(draw, min_size=0, max_size=100):
- min_date = pd.Timestamp("1900-01-01").to_pydatetime()
- max_date = pd.Timestamp("2099-12-31").to_pydatetime()
- start = draw(datetimes(min_value=min_date, max_value=max_date))
- periods = draw(integers(min_value=min_size, max_value=max_size))
- freq = draw(sampled_from(["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"]))
- return pd.date_range(start, periods=periods, freq=freq)
-
-
-@composite
-def frequencyStrings(draw, _):
- freq = draw(sampled_from(["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"]))
- mult = draw(integers(min_value=1, max_value=10))
- value = f"{mult}{freq}"
- return value
-
-@composite
-def dataFieldFlagger(draw):
- data = draw(dioses())
- field = draw(sampled_from(sorted(data.columns)))
- flagger = draw(flaggers(data))
- return data, field, flagger
-
-
-@composite
-def functionCalls(draw, module: str=None):
- func = draw(functions(module))
- kwargs = draw(functionKwargs(func))
- return func, kwargs
-
-
-@composite
-def functionKwargs(draw, func: SaQCFunction):
- data = draw(dioses())
- field = draw(sampled_from(sorted(data.columns)))
-
- kwargs = {
- "data": data,
- "field": field,
- "flagger": draw(flaggers(data))
- }
-
- column_name_strategy = lambda _: sampled_from(sorted(c for c in data.columns if c != field))
- interger_window_strategy = lambda _: integers(min_value=1, max_value=len(data[field]) - 1)
-
- register_type_strategy(FreqString, frequencyStrings)
- register_type_strategy(ColumnName, column_name_strategy)
- register_type_strategy(IntegerWindow, interger_window_strategy)
-
- for k, v in get_type_hints(func.func).items():
- if k not in {"data", "field", "flagger", "return"}:
- value = draw(from_type(v))
- # if v is TimestampColumnName:
- # value = draw(columnNames())
- # # we don't want to overwrite 'field'
- # assume(value != field)
- # # let's generate and add a timestamp column
- # data[value] = draw(dataSeries(dtypes="datetime64[ns]", length=len(data[field])))
- # # data[value] = draw(dataSeries(dtypes="datetime64[ns]"))
- kwargs[k] = value
-
- del _global_type_lookup[FreqString]
- del _global_type_lookup[ColumnName]
- del _global_type_lookup[IntegerWindow]
-
- return kwargs
diff --git a/testsfuzzy/__init__.py b/testsfuzzy/__init__.py
new file mode 100644
index 000000000..4265cc3e6
--- /dev/null
+++ b/testsfuzzy/__init__.py
@@ -0,0 +1 @@
+#!/usr/bin/env python
diff --git a/testsfuzzy/init.py b/testsfuzzy/init.py
new file mode 100644
index 000000000..2dee99665
--- /dev/null
+++ b/testsfuzzy/init.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python
+
+
+import numbers
+import dios
+import numpy as np
+import pandas as pd
+from typing import get_type_hints
+
+from hypothesis.strategies import (
+ lists,
+ sampled_from,
+ composite,
+ from_regex,
+ sampled_from,
+ datetimes,
+ integers,
+ register_type_strategy,
+ from_type,
+)
+from hypothesis.extra.numpy import arrays, from_dtype
+from hypothesis.strategies._internal.types import _global_type_lookup
+
+from saqc.common import *
+from saqc.core.register import FUNC_MAP
+from saqc.core.lib import SaQCFunction
+from saqc.lib.types import FreqString, ColumnName, IntegerWindow
+from saqc.flagger import Flagger, initFlagsLike
+
+MAX_EXAMPLES = 50
+
+
+# MAX_EXAMPLES = 100000
+
+
+@composite
+def dioses(draw, min_cols=1):
+ """
+ initialize data according to the current restrictions
+ """
+ # NOTE:
+ # The following restriction showed up and should be enforced during init:
+ # - Column names need to satisify the following regex: [A-Za-z0-9_-]+
+ # - DatetimeIndex needs to be sorted
+ # - Integer values larger than 2**53 lead to numerical instabilities during
+ # the integer->float->integer type conversion in _maskData/_unmaskData.
+
+ cols = draw(lists(columnNames(), unique=True, min_size=min_cols))
+ columns = {
+ c: draw(dataSeries(min_size=3))
+ for c in cols
+ }
+ return dios.DictOfSeries(columns)
+
+
+@composite
+def dataSeries(draw, min_size=0, max_size=100, dtypes=("float32", "float64", "int32", "int64")):
+ if np.isscalar(dtypes):
+ dtypes = (dtypes,)
+
+ dtype = np.dtype(draw(sampled_from(dtypes)))
+ if issubclass(dtype.type, numbers.Integral):
+ info = np.iinfo(dtype)
+ elif issubclass(dtype.type, numbers.Real):
+ info = np.finfo(dtype)
+ else:
+ raise ValueError("only numerical dtypes are supported")
+ # we don't want to fail just because of overflows
+ elements = from_dtype(dtype, min_value=info.min + 1, max_value=info.max - 1)
+
+ index = draw(daterangeIndexes(min_size=min_size, max_size=max_size))
+ values = draw(arrays(dtype=dtype, elements=elements, shape=len(index)))
+ return pd.Series(data=values, index=index)
+
+
+@composite
+def columnNames(draw):
+ return draw(from_regex(r"[A-Za-z0-9_-]+", fullmatch=True))
+
+
+@composite
+def flaggers(draw, data):
+ """
+ initialize a flagger and set some flags
+ """
+ flagger = initFlagsLike(data)
+ for col, srs in data.items():
+ loc_st = lists(sampled_from(sorted(srs.index)), unique=True, max_size=len(srs) - 1)
+ flagger[draw(loc_st), col] = BAD
+ return flagger
+
+
+@composite
+def functions(draw, module: str = None):
+ samples = tuple(FUNC_MAP.values())
+ if module:
+ samples = tuple(f for f in samples if f.name.startswith(module))
+ # samples = [FUNC_MAP["drift.correctExponentialDrift"]]
+ return draw(sampled_from(samples))
+
+
+@composite
+def daterangeIndexes(draw, min_size=0, max_size=100):
+ min_date = pd.Timestamp("1900-01-01").to_pydatetime()
+ max_date = pd.Timestamp("2099-12-31").to_pydatetime()
+ start = draw(datetimes(min_value=min_date, max_value=max_date))
+ periods = draw(integers(min_value=min_size, max_value=max_size))
+ freq = draw(sampled_from(["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"]))
+ return pd.date_range(start, periods=periods, freq=freq)
+
+
+@composite
+def frequencyStrings(draw, _):
+ freq = draw(sampled_from(["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"]))
+ mult = draw(integers(min_value=1, max_value=10))
+ value = f"{mult}{freq}"
+ return value
+
+
+@composite
+def dataFieldFlagger(draw):
+ data = draw(dioses())
+ field = draw(sampled_from(sorted(data.columns)))
+ flagger = draw(flaggers(data))
+ return data, field, flagger
+
+
+@composite
+def functionCalls(draw, module: str = None):
+ func = draw(functions(module))
+ kwargs = draw(functionKwargs(func))
+ return func, kwargs
+
+
+@composite
+def functionKwargs(draw, func: SaQCFunction):
+ data = draw(dioses())
+ field = draw(sampled_from(sorted(data.columns)))
+
+ kwargs = {
+ "data": data,
+ "field": field,
+ "flagger": draw(flaggers(data))
+ }
+
+ column_name_strategy = lambda _: sampled_from(sorted(c for c in data.columns if c != field))
+ interger_window_strategy = lambda _: integers(min_value=1, max_value=len(data[field]) - 1)
+
+ register_type_strategy(FreqString, frequencyStrings)
+ register_type_strategy(ColumnName, column_name_strategy)
+ register_type_strategy(IntegerWindow, interger_window_strategy)
+
+ for k, v in get_type_hints(func.func).items():
+ if k not in {"data", "field", "flagger", "return"}:
+ value = draw(from_type(v))
+ # if v is TimestampColumnName:
+ # value = draw(columnNames())
+ # # we don't want to overwrite 'field'
+ # assume(value != field)
+ # # let's generate and add a timestamp column
+ # data[value] = draw(dataSeries(dtypes="datetime64[ns]", length=len(data[field])))
+ # # data[value] = draw(dataSeries(dtypes="datetime64[ns]"))
+ kwargs[k] = value
+
+ del _global_type_lookup[FreqString]
+ del _global_type_lookup[ColumnName]
+ del _global_type_lookup[IntegerWindow]
+
+ return kwargs
diff --git a/test/funcs/test_functions_fuzzy.py b/testsfuzzy/test_functions.py
similarity index 63%
rename from test/funcs/test_functions_fuzzy.py
rename to testsfuzzy/test_functions.py
index 905a28611..dd86b7ab4 100644
--- a/test/funcs/test_functions_fuzzy.py
+++ b/testsfuzzy/test_functions.py
@@ -1,14 +1,12 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from saqc.core.register import FUNC_MAP
from hypothesis import given, settings
-from hypothesis.strategies import (
- data,
-)
+from hypothesis.strategies import data
-from test.common import MAX_EXAMPLES, functionKwargs
+from saqc.core.register import FUNC_MAP
+from testsfuzzy.init import MAX_EXAMPLES, functionKwargs
@settings(max_examples=MAX_EXAMPLES, deadline=None)
@@ -23,12 +21,15 @@ def callWontBreak(drawer, func_name: str):
# ------
# NOTE:
-# needs a more alaborated test, as it calls into
+# needs a more elaborated test, as it calls into
# `changepoints.assignChangePointClusters`
-# def test_breaks_flagJumps():
-# callWontBreak("breaks.flagJumps")
-# def test_breaks_flagIsolated():
-# callWontBreak("breaks.flagIsolated")
+def test_breaks_flagJumps():
+ callWontBreak("breaks.flagJumps")
+
+
+def test_breaks_flagIsolated():
+ callWontBreak("breaks.flagIsolated")
+
def test_breaks_flagMissing():
callWontBreak("breaks.flagMissing")
@@ -40,6 +41,7 @@ def test_breaks_flagMissing():
def test_constats_flagConstats():
callWontBreak("constants.flagConstants")
+
def test_constants_flagByVariance():
callWontBreak("constants.flagByVariance")
@@ -50,48 +52,58 @@ def test_constants_flagByVariance():
def test_flagtools_clearFlags():
callWontBreak("flagtools.clearFlags")
+
def test_flagtools_forceFlags():
callWontBreak("flagtools.clearFlags")
+
# NOTE:
# all of the following tests fail to sample data for `flag=typing.Any`
# with the new flagger in place this should be easy to fix
-# def test_flagtools_flagGood():
-# callWontBreak("flagtools.flagGood")
+def test_flagtools_flagGood():
+ callWontBreak("flagtools.flagGood")
-# def test_flagtools_flagUnflagged():
-# callWontBreak("flagtools.flagUnflagged")
-# def test_flagtools_flagManual():
-# callWontBreak("flagtools.flagManual")
+def test_flagtools_flagUnflagged():
+ callWontBreak("flagtools.flagUnflagged")
+
+
+def test_flagtools_flagManual():
+ callWontBreak("flagtools.flagManual")
# outliers
# --------
# NOTE: needs a more elaborated test, I guess
-# def test_outliers_flagByStray():
-# callWontBreak("outliers.flagByStray")
+def test_outliers_flagByStray():
+ callWontBreak("outliers.flagByStray")
+
# NOTE: fails in a strategy, maybe `Sequence[ColumnName]`
-# def test_outliers_flagMVScores():
-# callWontBreak("outliers.flagMVScores")
+def test_outliers_flagMVScores():
+ callWontBreak("outliers.flagMVScores")
+
# NOTE:
# fails as certain combinations of frquency strings don't make sense
# a more elaborate test is needed
-# def test_outliers_flagRaise():
-# callWontBreak("outliers.flagRaise")
+def test_outliers_flagRaise():
+ callWontBreak("outliers.flagRaise")
+
def test_outliers_flagMAD():
callWontBreak("outliers.flagMAD")
+
def test_outliers_flagByGrubbs():
callWontBreak("outliers.flagByGrubbs")
+
def test_outliers_flagRange():
callWontBreak("outliers.flagRange")
+
# NOTE: fails in a strategy, maybe `Sequence[ColumnName]`
-# def test_outliers_flagCrossStatistic():
-# callWontBreak("outliers.flagCrossStatistic")
+def test_outliers_flagCrossStatistic():
+ callWontBreak("outliers.flagCrossStatistic")
diff --git a/test/core/test_masking.py b/testsfuzzy/test_masking.py
similarity index 98%
rename from test/core/test_masking.py
rename to testsfuzzy/test_masking.py
index 6236a55e2..b1eb5861e 100644
--- a/test/core/test_masking.py
+++ b/testsfuzzy/test_masking.py
@@ -16,7 +16,7 @@ from saqc.common import *
from saqc.flagger import Flagger, initFlagsLike
from saqc.core.register import _maskData, _unmaskData
-from test.common import dataFieldFlagger, MAX_EXAMPLES
+from testsfuzzy.init import dataFieldFlagger, MAX_EXAMPLES
logging.disable(logging.CRITICAL)
--
GitLab
From 80b82fb17f8e703b16a0580c565d40f89cd79675 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 12 Mar 2021 14:32:13 +0100
Subject: [PATCH 039/180] fixed (generic) isflagged and its test
---
saqc/funcs/generic.py | 26 ++++++++-
test/funcs/test_generic_config_functions.py | 63 +++++++++++----------
2 files changed, 57 insertions(+), 32 deletions(-)
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index ed64f7ca0..862b3b981 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -21,13 +21,33 @@ _OP = {'<': op.lt, '<=': op.le, '==': op.eq, '!=': op.ne, '>': op.gt, '>=': op.g
def _dslIsFlagged(
- flagger: Flagger, var: pd.Series, flag: float = UNFLAGGED, comparator: str = ">"
+ flagger: Flagger, var: pd.Series, flag: float = None, comparator: str = None
) -> Union[pd.Series, DictOfSeries]:
"""
helper function for `flag`
+
+ Param Combinations
+ ------------------
+ - ``isflagged('var')`` : show me (anything) flagged
+ - ``isflagged('var', DOUBT)`` : show me ``flags >= DOUBT``
+ - ``isflagged('var', DOUBT, '==')`` : show me ``flags == DOUBT``
+
+ Raises
+ ------
+ ValueError: if `comparator` is passed but no `flag` vaule. Eg. ``isflagged('var', comparator='>=')``
"""
- comparison = _OP[comparator]
- return comparison(flagger[var.name], flag)
+ if flag is None:
+ if comparator is not None:
+ raise ValueError('if `comparator` is used, explicitly pass a `flag` level.')
+ flag = UNFLAGGED
+ comparator = '>'
+
+ # default
+ if comparator is None:
+ comparator = '>='
+
+ _op = _OP[comparator]
+ return _op(flagger[var.name], flag)
def _execGeneric(flagger: Flagger, data: DictOfSeries, func: Callable[[pd.Series], pd.Series], field: str,
diff --git a/test/funcs/test_generic_config_functions.py b/test/funcs/test_generic_config_functions.py
index 7677c3c27..81e91d643 100644
--- a/test/funcs/test_generic_config_functions.py
+++ b/test/funcs/test_generic_config_functions.py
@@ -2,22 +2,20 @@
# -*- coding: utf-8 -*-
import ast
-
import pytest
import numpy as np
import pandas as pd
-
-from dios import DictOfSeries
-
-from test.common import TESTFLAGGER, TESTNODATA, initData, writeIO
+import dios
from saqc.common import *
from saqc.flagger import Flagger, initFlagsLike
from saqc.core.visitor import ConfigFunctionParser
from saqc.core.config import Fields as F
from saqc.core.register import register
-from saqc import SaQC
from saqc.funcs.generic import _execGeneric
+from saqc import SaQC
+
+from test.common import TESTNODATA, initData, writeIO
@pytest.fixture
@@ -32,7 +30,7 @@ def data_diff():
col1 = data[data.columns[1]]
mid = len(col0) // 2
offset = len(col0) // 8
- return DictOfSeries(data={col0.name: col0.iloc[: mid + offset], col1.name: col1.iloc[mid - offset :],})
+ return dios.DictOfSeries(data={col0.name: col0.iloc[: mid + offset], col1.name: col1.iloc[mid - offset :],})
def _compileGeneric(expr, flagger):
@@ -41,8 +39,8 @@ def _compileGeneric(expr, flagger):
return kwargs["func"]
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_missingIdentifier(data, flagger):
+def test_missingIdentifier(data):
+ flagger = Flagger()
# NOTE:
# - the error is only raised at runtime during parsing would be better
@@ -57,9 +55,8 @@ def test_missingIdentifier(data, flagger):
_execGeneric(flagger, data, func, field="", nodata=np.nan)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_syntaxError(flagger):
-
+def test_syntaxError():
+ flagger = Flagger()
tests = [
"range(x=5",
"rangex=5)",
@@ -106,8 +103,7 @@ def test_comparisonOperators(data):
assert np.all(result == expected)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_arithmeticOperators(data, flagger):
+def test_arithmeticOperators(data):
flagger = initFlagsLike(data)
var1, *_ = data.columns
this = data[var1]
@@ -127,8 +123,7 @@ def test_arithmeticOperators(data, flagger):
assert np.all(result == expected)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_nonReduncingBuiltins(data, flagger):
+def test_nonReduncingBuiltins(data):
flagger = initFlagsLike(data)
var1, *_ = data.columns
this = var1
@@ -147,10 +142,8 @@ def test_nonReduncingBuiltins(data, flagger):
assert (result == expected).all()
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
@pytest.mark.parametrize("nodata", TESTNODATA)
-def test_reduncingBuiltins(data, flagger, nodata):
-
+def test_reduncingBuiltins(data, nodata):
data.loc[::4] = nodata
flagger = initFlagsLike(data)
var1 = data.columns[0]
@@ -171,10 +164,10 @@ def test_reduncingBuiltins(data, flagger, nodata):
assert result == expected
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
@pytest.mark.parametrize("nodata", TESTNODATA)
-def test_ismissing(data, flagger, nodata):
+def test_ismissing(data, nodata):
+ flagger = initFlagsLike(data)
data.iloc[: len(data) // 2, 0] = np.nan
data.iloc[(len(data) // 2) + 1 :, 0] = -9999
this = data.iloc[:, 0]
@@ -190,9 +183,8 @@ def test_ismissing(data, flagger, nodata):
assert np.all(result == expected)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
@pytest.mark.parametrize("nodata", TESTNODATA)
-def test_bitOps(data, flagger, nodata):
+def test_bitOps(data, nodata):
var1, var2, *_ = data.columns
this = var1
@@ -220,14 +212,26 @@ def test_isflagged(data):
(f"isflagged({var1})", flagger[var1] > UNFLAGGED),
(f"isflagged({var1}, flag=BAD)", flagger[var1] >= BAD),
(f"isflagged({var1}, UNFLAGGED, '==')", flagger[var1] == UNFLAGGED),
- (f"~isflagged({var2})", ~(flagger[var2] > UNFLAGGED)),
- (f"~({var2}>999) & (~isflagged({var2}))", ~(data[var2] > 999) & ~(flagger[var2] > UNFLAGGED)),
+ (f"~isflagged({var2})", flagger[var2] == UNFLAGGED),
+ (f"~({var2}>999) & (~isflagged({var2}))", ~(data[var2] > 999) & (flagger[var2] == UNFLAGGED)),
]
- for test, expected in tests:
- func = _compileGeneric(f"generic.flag(func={test}, flag=BAD)", flagger)
- result = _execGeneric(flagger, data, func, field=None, nodata=np.nan)
- assert np.all(result == expected)
+ for i, (test, expected) in enumerate(tests):
+ try:
+ func = _compileGeneric(f"generic.flag(func={test}, flag=BAD)", flagger)
+ result = _execGeneric(flagger, data, func, field=None, nodata=np.nan)
+ assert np.all(result == expected)
+ except Exception:
+ print(i, test)
+ raise
+
+ # test bad combination
+ for comp in ['>', '>=', '==', '!=', '<', '<=']:
+ fails = f"isflagged({var1}, comparator='{comp}')"
+
+ func = _compileGeneric(f"generic.flag(func={fails}, flag=BAD)", flagger)
+ with pytest.raises(ValueError):
+ _execGeneric(flagger, data, func, field=None, nodata=np.nan)
def test_variableAssignments(data):
@@ -249,6 +253,7 @@ def test_variableAssignments(data):
assert set(result_flagger.columns) == set(data.columns) | {"dummy1", "dummy2"}
+# TODO: why this must(!) fail ? - a comment would be helpful
@pytest.mark.xfail(strict=True)
def test_processMultiple(data_diff):
var1, var2, *_ = data_diff.columns
--
GitLab
From 04ebf7fd762f74202c9b28d0d3ce0a957ac3d37b Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 12 Mar 2021 15:18:40 +0100
Subject: [PATCH 040/180] fixed fuzzy tests
---
testsfuzzy/init.py | 2 --
testsfuzzy/test_functions.py | 54 ++++++++++++++++++++----------------
2 files changed, 30 insertions(+), 26 deletions(-)
diff --git a/testsfuzzy/init.py b/testsfuzzy/init.py
index 2dee99665..adbbffdc5 100644
--- a/testsfuzzy/init.py
+++ b/testsfuzzy/init.py
@@ -28,8 +28,6 @@ from saqc.lib.types import FreqString, ColumnName, IntegerWindow
from saqc.flagger import Flagger, initFlagsLike
MAX_EXAMPLES = 50
-
-
# MAX_EXAMPLES = 100000
diff --git a/testsfuzzy/test_functions.py b/testsfuzzy/test_functions.py
index dd86b7ab4..fc3caa00e 100644
--- a/testsfuzzy/test_functions.py
+++ b/testsfuzzy/test_functions.py
@@ -3,7 +3,7 @@
from hypothesis import given, settings
-from hypothesis.strategies import data
+from hypothesis.strategies import data, from_type
from saqc.core.register import FUNC_MAP
from testsfuzzy.init import MAX_EXAMPLES, functionKwargs
@@ -14,6 +14,11 @@ from testsfuzzy.init import MAX_EXAMPLES, functionKwargs
def callWontBreak(drawer, func_name: str):
func = FUNC_MAP[func_name]
kwargs = drawer.draw(functionKwargs(func))
+
+ # TODO: workaround until `flag` is explicitly exposed in signature
+ flag = drawer.draw(from_type(float))
+ kwargs.setdefault('flag', flag)
+
func(**kwargs)
@@ -68,42 +73,43 @@ def test_flagtools_flagUnflagged():
callWontBreak("flagtools.flagUnflagged")
-def test_flagtools_flagManual():
- callWontBreak("flagtools.flagManual")
+# NOTE: the problem is `mflag` which can be Any
+# def test_flagtools_flagManual():
+# callWontBreak("flagtools.flagManual")
# outliers
# --------
-
+#
# NOTE: needs a more elaborated test, I guess
-def test_outliers_flagByStray():
- callWontBreak("outliers.flagByStray")
+# def test_outliers_flagByStray():
+# callWontBreak("outliers.flagByStray")
# NOTE: fails in a strategy, maybe `Sequence[ColumnName]`
-def test_outliers_flagMVScores():
- callWontBreak("outliers.flagMVScores")
+# def test_outliers_flagMVScores():
+# callWontBreak("outliers.flagMVScores")
# NOTE:
# fails as certain combinations of frquency strings don't make sense
# a more elaborate test is needed
-def test_outliers_flagRaise():
- callWontBreak("outliers.flagRaise")
-
-
-def test_outliers_flagMAD():
- callWontBreak("outliers.flagMAD")
-
-
-def test_outliers_flagByGrubbs():
- callWontBreak("outliers.flagByGrubbs")
-
-
-def test_outliers_flagRange():
- callWontBreak("outliers.flagRange")
+# def test_outliers_flagRaise():
+# callWontBreak("outliers.flagRaise")
+#
+#
+# def test_outliers_flagMAD():
+# callWontBreak("outliers.flagMAD")
+#
+#
+# def test_outliers_flagByGrubbs():
+# callWontBreak("outliers.flagByGrubbs")
+#
+#
+# def test_outliers_flagRange():
+# callWontBreak("outliers.flagRange")
# NOTE: fails in a strategy, maybe `Sequence[ColumnName]`
-def test_outliers_flagCrossStatistic():
- callWontBreak("outliers.flagCrossStatistic")
+# def test_outliers_flagCrossStatistic():
+# callWontBreak("outliers.flagCrossStatistic")
--
GitLab
From 378c030bdd224057d361f0987f5577cf86731889 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 12 Mar 2021 15:50:31 +0100
Subject: [PATCH 041/180] fixed/cleand test
---
test/core/test_core.py | 4 +-
test/core/test_reader.py | 7 +-
test/{funcs/conftest.py => fixtures.py} | 72 +++++++++-----
test/flagger/test_dmpflagger.py | 117 -----------------------
test/flagger/test_flagger.py | 3 +
test/flagger/test_flags.py | 25 +++--
test/flagger/test_history.py | 1 +
test/flagger/test_positionalflagger.py | 57 -----------
test/funcs/test_constants_detection.py | 4 +-
test/funcs/test_functions.py | 23 +++--
test/funcs/test_generic_api_functions.py | 3 +-
test/funcs/test_harm_funcs.py | 3 +-
test/funcs/test_modelling.py | 15 ++-
test/funcs/test_pattern_rec.py | 4 +-
test/funcs/test_proc_functions.py | 9 +-
test/funcs/test_spikes_detection.py | 9 +-
test/run_pytest.py | 4 -
17 files changed, 114 insertions(+), 246 deletions(-)
rename test/{funcs/conftest.py => fixtures.py} (74%)
delete mode 100644 test/flagger/test_dmpflagger.py
delete mode 100644 test/flagger/test_positionalflagger.py
delete mode 100644 test/run_pytest.py
diff --git a/test/core/test_core.py b/test/core/test_core.py
index 5527f2ee2..51b99f5c4 100644
--- a/test/core/test_core.py
+++ b/test/core/test_core.py
@@ -2,19 +2,17 @@
# -*- coding: utf-8 -*-
import logging
-
import pytest
import numpy as np
import pandas as pd
-
from saqc.common import *
from saqc.flagger import Flagger, initFlagsLike
from saqc.funcs import flagRange
from saqc.lib import plotting as splot
-from test.common import initData, TESTFLAGGER, flagAll
from saqc import SaQC, register
+from test.common import initData, flagAll
# no logging output needed here
# -> can this be configured on the test runner level?
diff --git a/test/core/test_reader.py b/test/core/test_reader.py
index ce8438eff..9ab7d2a51 100644
--- a/test/core/test_reader.py
+++ b/test/core/test_reader.py
@@ -1,19 +1,18 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from pathlib import Path
-
import pytest
import numpy as np
import pandas as pd
import dios
+from pathlib import Path
from saqc.core.config import Fields as F
-from test.common import initData, writeIO
-
from saqc.core.core import SaQC
from saqc.core.register import FUNC_MAP, register
+from test.common import initData, writeIO
+
@pytest.fixture
def data() -> dios.DictOfSeries:
diff --git a/test/funcs/conftest.py b/test/fixtures.py
similarity index 74%
rename from test/funcs/conftest.py
rename to test/fixtures.py
index abecdd3f2..8449ef6fa 100644
--- a/test/funcs/conftest.py
+++ b/test/fixtures.py
@@ -5,6 +5,11 @@ import pandas as pd
from dios import DictOfSeries
+# TODO: this is odd
+# Why not simple fixtures with talking-names,
+# that also take parameter, if needed
+
+
@pytest.fixture
def char_dict():
return {
@@ -18,10 +23,13 @@ def char_dict():
@pytest.fixture
def course_1(char_dict):
- # MONOTONOUSLY ASCENDING/DESCENDING
- # values , that monotonously ascend towards a peak level, and thereafter do monotonously decrease
- # the resulting drop/raise per value equals: (peak_level - initial_level) / (0.5*(periods-2))
- # periods number better be even!
+ """
+ MONOTONOUSLY ASCENDING/DESCENDING
+
+ values , that monotonously ascend towards a peak level, and thereafter do monotonously decrease
+ the resulting drop/raise per value equals: (peak_level - initial_level) / (0.5*(periods-2))
+ periods number better be even!
+ """
def fix_funk(
freq="10min",
periods=10,
@@ -48,10 +56,14 @@ def course_1(char_dict):
@pytest.fixture
def course_2(char_dict):
+ """
+ SINGLE_SPIKE
+
+ values , that linearly develop over the whole timeseries, from "initial_level" to "final_level", exhibiting
+ one "anomalous" or "outlierish" value of magnitude "out_val" at position "periods/2"
+ number of periods better be even!
+ """
# SINGLE_SPIKE
- # values , that linearly develop over the whole timeseries, from "initial_level" to "final_level", exhibiting
- # one "anomalous" or "outlierish" value of magnitude "out_val" at position "periods/2"
- # number of periods better be even!
def fix_funk(
freq="10min",
periods=10,
@@ -83,7 +95,11 @@ def course_2(char_dict):
@pytest.fixture
def course_test(char_dict):
- # Test function for pattern detection - same as test pattern for first three values, than constant function
+ """
+ Test function for pattern detection
+
+ same as test pattern for first three values, than constant function
+ """
def fix_funk(freq='1 D',
initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0), out_val=5, char_dict=char_dict):
t_index = pd.date_range(initial_index, freq=freq, periods=100)
@@ -100,13 +116,16 @@ def course_test(char_dict):
@pytest.fixture
def course_3(char_dict):
- # CROWD IN A PIT/CROWD ON A SUMMIT
- # values , that linearly develop over the whole timeseries, from "initial_level" to "final_level", exhibiting
- # a "crowd" of "anomalous" or "outlierish" values of magnitude "out_val".
- # The "crowd/group" of anomalous values starts at position "periods/2" and continues with an additional amount
- # of "crowd_size" values, that are each spaced "crowd_spacing" minutes from there predecessors.
- # number of periods better be even!
- # chrowd_size * crowd_spacing better be less then freq[minutes].
+ """
+ CROWD IN A PIT/CROWD ON A SUMMIT
+
+ values , that linearly develop over the whole timeseries, from "initial_level" to "final_level", exhibiting
+ a "crowd" of "anomalous" or "outlierish" values of magnitude "out_val".
+ The "crowd/group" of anomalous values starts at position "periods/2" and continues with an additional amount
+ of "crowd_size" values, that are each spaced "crowd_spacing" minutes from there predecessors.
+ number of periods better be even!
+ chrowd_size * crowd_spacing better be less then freq[minutes].
+ """
def fix_funk(
freq="10min",
periods=10,
@@ -148,9 +167,13 @@ def course_3(char_dict):
@pytest.fixture
def course_4(char_dict):
- # TEETH (ROW OF SPIKES) values , that remain on value level "base_level" and than begin exposing an outlierish or
- # spikey value of magnitude "out_val" every second timestep, starting at periods/2, with the first spike. number
- # of periods better be even!
+ """
+ TEETH (ROW OF SPIKES) values
+
+ , that remain on value level "base_level" and than begin exposing an outlierish or
+ spikey value of magnitude "out_val" every second timestep, starting at periods/2, with the first spike. number
+ of periods better be even!
+ """
def fix_funk(
freq="10min",
@@ -174,11 +197,14 @@ def course_4(char_dict):
@pytest.fixture
def course_5(char_dict):
- # NAN_holes
- # values , that ascend from initial_level to final_level linearly and have missing data(=nan)
- # at posiiotns "nan_slice", (=a slice or a list, for iloc indexing)
- # periods better be even!
- # periods better be greater 5
+ """
+ NAN_holes
+
+ values , that ascend from initial_level to final_level linearly and have missing data(=nan)
+ at positions "nan_slice", (=a slice or a list, for iloc indexing)
+ periods better be even!
+ periods better be greater 5
+ """
def fix_funk(
freq="10min",
diff --git a/test/flagger/test_dmpflagger.py b/test/flagger/test_dmpflagger.py
deleted file mode 100644
index 677f54cbe..000000000
--- a/test/flagger/test_dmpflagger.py
+++ /dev/null
@@ -1,117 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import json
-
-import numpy as np
-import pandas as pd
-import pytest
-
-from test.common import initData
-
-DmpFlagger = NotImplemented
-pytest.skip("DmpFlagger is deprecated.", allow_module_level=True)
-
-
-@pytest.fixture
-def data():
- return initData(cols=1)
-
-
-@pytest.fixture
-def data_4cols():
- return initData(cols=4)
-
-
-def parseComments(data):
- return np.array([json.loads(v)["comment"] for v in data.to_df().values.flatten()])
-
-
-def test_initFlags(data):
- flagger = DmpFlagger().initFlags(data=data)
- assert (flagger._flags == flagger.UNFLAGGED).all(axis=None)
- assert (flagger._causes == "").all(axis=None)
- assert (flagger._comments == "").all(axis=None)
-
-
-def test_mergeFlaggerOuter(data):
-
- flagger = DmpFlagger()
-
- field = data.columns[0]
-
- data_left = data
-
- data_right = data.to_df()
- dates = data_right.index.to_series()
- dates[len(dates) // 2 :] += pd.Timedelta("1Min")
- data_right.index = dates
- data_right = data_right.to_dios()
-
- left = flagger.initFlags(data=data_left).setFlags(
- field=field, flag=flagger.BAD, cause="SaQCLeft", comment="testLeft"
- )
-
- right = flagger.initFlags(data=data_right).setFlags(
- field=field, flag=flagger.GOOD, cause="SaQCRight", comment="testRight"
- )
-
- merged = left.merge(right, join="outer")
-
- right_index = data_right[field].index.difference(data_left[field].index)
- assert (merged._flags.loc[right_index] == flagger.GOOD).all(axis=None)
- assert (merged._causes.loc[right_index] == "SaQCRight").all(axis=None)
- assert np.all(parseComments(merged._comments.loc[right_index]) == "testRight")
-
- left_index = data_left[field].index
- assert (merged._flags.loc[left_index] == flagger.BAD).all(axis=None)
- assert (merged._causes.loc[left_index] == "SaQCLeft").all(axis=None)
- assert np.all(parseComments(merged._comments.loc[left_index]) == "testLeft")
-
-
-def test_mergeFlaggerInner(data):
-
- flagger = DmpFlagger()
-
- field = data.columns[0]
-
- data_left = data
- data_right = data.iloc[::2]
-
- left = flagger.initFlags(data=data_left).setFlags(
- field=field, flag=flagger.BAD, cause="SaQCLeft", comment="testLeft"
- )
-
- right = flagger.initFlags(data=data_right).setFlags(
- field=field, flag=flagger.GOOD, cause="SaQCRight", comment="testRight"
- )
-
- merged = left.merge(right, join="inner")
-
- assert (merged._flags[field].index == data_right[field].index).all()
- assert (merged._causes[field].index == data_right[field].index).all()
- assert (merged._comments[field].index == data_right[field].index).all()
-
- assert (merged._flags[field] == flagger.BAD).all()
- assert (merged._causes[field] == "SaQCLeft").all(axis=None)
- assert np.all(parseComments(merged._comments) == "testLeft")
-
-
-def test_sliceFlaggerDrop(data):
- flagger = DmpFlagger().initFlags(data)
- with pytest.raises(TypeError):
- flagger.getFlags(field=data.columns, drop="var")
-
- field = data.columns[0]
- expected = data[data.columns.drop(field)].to_df()
-
- filtered = flagger.slice(drop=field)
-
- assert (filtered._flags.columns == expected.columns).all(axis=None)
- assert (filtered._comments.columns == expected.columns).all(axis=None)
- assert (filtered._causes.columns == expected.columns).all(axis=None)
-
- assert (filtered._flags.to_df().index == expected.index).all(axis=None)
- assert (filtered._comments.to_df().index == expected.index).all(axis=None)
- assert (filtered._causes.to_df().index == expected.index).all(axis=None)
-
diff --git a/test/flagger/test_flagger.py b/test/flagger/test_flagger.py
index 158baaae8..a304409fb 100644
--- a/test/flagger/test_flagger.py
+++ b/test/flagger/test_flagger.py
@@ -10,6 +10,9 @@ import dios
from test.common import TESTFLAGGER, initData
+pytestmark = pytest.mark.skip('old flagger tests - rewrite needed')
+
+
def _getDataset(rows, cols):
return initData(cols=cols, rows=rows, start_date="2011-01-01", end_date="2011-01-10")
diff --git a/test/flagger/test_flags.py b/test/flagger/test_flags.py
index 83c156011..c04177101 100644
--- a/test/flagger/test_flags.py
+++ b/test/flagger/test_flags.py
@@ -4,12 +4,13 @@ import pytest
import numpy as np
import pandas as pd
-from saqc import BAD, UNFLAGGED
+from saqc.common import *
+from saqc.flagger.flags import Flags
+
from test.flagger.test_history import (
History,
is_equal as hist_equal,
)
-from saqc.flagger.flags import Flags
_data = [
@@ -181,11 +182,17 @@ def test_set_flags_with_mask(data: np.array):
assert all(flags[c].loc[mask] == 444.)
assert all(flags[c].loc[~mask] != 444.)
- # test length miss-match
- if len(vector):
- vector = vector[:-1]
+ # test length miss-match (mask)
+ if len(mask) > 1:
+ wrong_len = mask[:-1]
with pytest.raises(ValueError):
- flags[mask, c] = vector
+ flags[wrong_len, c] = vector
+
+ # test length miss-match (value)
+ if len(vector) > 1:
+ wrong_len = vector[:-1]
+ with pytest.raises(ValueError):
+ flags[mask, c] = wrong_len
@pytest.mark.parametrize('data', data)
@@ -215,6 +222,12 @@ def test_set_flags_with_index(data: np.array):
assert all(flags[c].loc[mask] == 444.)
assert all(flags[c].loc[~mask] != 444.)
+ # test length miss-match (value)
+ if len(vector) > 1:
+ wrong_len = vector[:-1]
+ with pytest.raises(ValueError):
+ flags[index, c] = wrong_len
+
def test_cache():
arr = np.array([
diff --git a/test/flagger/test_history.py b/test/flagger/test_history.py
index d6a084827..5a95585c0 100644
--- a/test/flagger/test_history.py
+++ b/test/flagger/test_history.py
@@ -3,6 +3,7 @@
import pytest
import numpy as np
import pandas as pd
+
from saqc.flagger.history import History
# see #GH143 combined backtrack
diff --git a/test/flagger/test_positionalflagger.py b/test/flagger/test_positionalflagger.py
deleted file mode 100644
index 45506a070..000000000
--- a/test/flagger/test_positionalflagger.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import pytest
-
-import numpy as np
-
-from test.common import initData
-
-PositionalFlagger = NotImplemented
-pytest.skip("PositionalFlagger is deprecated.", allow_module_level=True)
-
-@pytest.fixture
-def data():
- return initData(cols=2)
-
-
-def test_initFlags(data):
- flagger = PositionalFlagger().initFlags(data=data)
- assert (flagger.isFlagged() == False).all(axis=None)
- assert (flagger.flags == flagger.UNFLAGGED).all(axis=None)
-
-
-def test_setFlags(data):
- flagger = PositionalFlagger().initFlags(data=data)
-
- field = data.columns[0]
- mask = np.zeros(len(data[field]), dtype=bool)
- mask[1:10:2] = True
-
- flagger = flagger.setFlags(field=field, loc=mask, flag=flagger.SUSPICIOUS)
- assert (flagger.flags.loc[mask, field] == "91").all(axis=None)
- assert (flagger.flags.loc[~mask, field] == "90").all(axis=None)
-
- flagger = flagger.setFlags(field=field, loc=~mask, flag=flagger.BAD)
- assert (flagger.flags.loc[~mask, field] == "902").all(axis=None)
- assert (flagger.flags.loc[mask, field] == "910").all(axis=None)
-
- assert (flagger.flags[data.columns[1]] == "-1").all(axis=None)
-
-
-def test_isFlagged(data):
- flagger = PositionalFlagger().initFlags(data=data)
- field = data.columns[0]
-
- mask_sus = np.zeros(len(data[field]), dtype=bool)
- mask_sus[1:20:2] = True
- flagger = flagger.setFlags(field=field, loc=mask_sus, flag=flagger.SUSPICIOUS)
- assert (flagger.isFlagged(field=field, comparator=">=", flag=flagger.SUSPICIOUS)[mask_sus] == True).all(axis=None)
- assert (flagger.isFlagged(field=field, comparator=">", flag=flagger.SUSPICIOUS) == False).all(axis=None)
-
- mask_bad = np.zeros(len(data[field]), dtype=bool)
- mask_bad[1:10:2] = True
- flagger = flagger.setFlags(field=field, loc=mask_bad, flag=flagger.BAD)
- assert (flagger.isFlagged(field=field, comparator=">")[mask_sus] == True).all(axis=None)
- assert (flagger.isFlagged(field=field, comparator=">=", flag=flagger.BAD)[mask_bad] == True).all(axis=None)
- assert (flagger.isFlagged(field=field, comparator=">", flag=flagger.BAD) == False).all(axis=None)
diff --git a/test/funcs/test_constants_detection.py b/test/funcs/test_constants_detection.py
index b7cabb50e..30a16db10 100644
--- a/test/funcs/test_constants_detection.py
+++ b/test/funcs/test_constants_detection.py
@@ -4,9 +4,11 @@
import pytest
import numpy as np
+from saqc.common import *
from saqc.funcs.constants import flagConstants, flagByVariance
+from saqc.flagger import initFlagsLike
-from test.common import initData, initFlagsLike, BAD
+from test.common import initData
@pytest.fixture
diff --git a/test/funcs/test_functions.py b/test/funcs/test_functions.py
index 1a56f8e8a..a205e517d 100644
--- a/test/funcs/test_functions.py
+++ b/test/funcs/test_functions.py
@@ -14,7 +14,9 @@ from saqc.funcs.flagtools import flagManual, forceFlags, clearFlags
from saqc.funcs.tools import drop, copy, mask
from saqc.funcs.resampling import reindexFlags
from saqc.funcs.breaks import flagIsolated
-from test.common import initData, TESTFLAGGER
+
+from test.fixtures import *
+from test.common import initData
@pytest.fixture
@@ -37,7 +39,6 @@ def test_flagRange(data, field):
def test_flagSesonalRange(data, field):
- # prepare
data.iloc[::2] = 0
data.iloc[1::2] = 50
nyears = len(data[field].index.year.unique())
@@ -84,27 +85,35 @@ def test_forceFlags(data, field):
assert all(flagger[field] == DOUBT)
-# TODO: @luenensc: i dont get the test -- palmb
def test_flagIsolated(data, field):
flagger = initFlagsLike(data)
data.iloc[1:3, 0] = np.nan
data.iloc[4:5, 0] = np.nan
+ flagger[data[field].index[5:6], field] = BAD
data.iloc[11:13, 0] = np.nan
data.iloc[15:17, 0] = np.nan
- s = data[field].iloc[5:6]
- flagger[s.index, field] = BAD
+ # data flags
+ # 2016-01-01 0.0 -inf
+ # 2016-01-02 NaN -inf
+ # 2016-01-03 NaN -inf
+ # 2016-01-04 3.0 -inf
+ # 2016-01-05 NaN -inf
+ # 2016-01-06 5.0 255.0
+ # 2016-01-07 6.0 -inf
+ # 2016-01-08 7.0 -inf
+ # .. .. ..
_, flagger_result = flagIsolated(data, field, flagger, group_window="1D", gap_window="2.1D", flag=BAD)
- assert flagger_result[field][slice(3, 6, 2)].all()
+ assert flagger_result[field].iloc[[3, 5]].all()
data, flagger_result = flagIsolated(
data, field, flagger_result,
group_window="2D", gap_window="2.1D", continuation_range="1.1D", flag=BAD
)
- assert flagger_result[field][[3, 5, 13, 14]].all()
+ assert flagger_result[field].iloc[[3, 5, 13, 14]].all()
@pytest.mark.parametrize("dat", [pytest.lazy_fixture("course_2")])
diff --git a/test/funcs/test_generic_api_functions.py b/test/funcs/test_generic_api_functions.py
index 950dbfd7f..76c9f0d53 100644
--- a/test/funcs/test_generic_api_functions.py
+++ b/test/funcs/test_generic_api_functions.py
@@ -8,11 +8,12 @@ import pandas as pd
from saqc.common import *
from saqc.core.register import register
-from saqc import SaQC
from saqc.funcs.tools import mask
+from saqc import SaQC
from test.common import initData, flagAll
+
register(masking='field')(flagAll)
diff --git a/test/funcs/test_harm_funcs.py b/test/funcs/test_harm_funcs.py
index 2fca105a6..187a4b7c4 100644
--- a/test/funcs/test_harm_funcs.py
+++ b/test/funcs/test_harm_funcs.py
@@ -2,9 +2,8 @@
# -*- coding: utf-8 -*-
-# see test/functs/conftest.py for global fixtures "course_..."
+# see test/functs/fixtures.py for global fixtures "course_..."
import pytest
-
import numpy as np
import pandas as pd
import dios
diff --git a/test/funcs/test_modelling.py b/test/funcs/test_modelling.py
index 00ba8a881..748a06fa3 100644
--- a/test/funcs/test_modelling.py
+++ b/test/funcs/test_modelling.py
@@ -2,20 +2,19 @@
# -*- coding: utf-8 -*-
-# see test/functs/conftest.py for global fixtures "course_..."
+# see test/functs/fixtures.py for global fixtures "course_..."
import pytest
-
import numpy as np
import pandas as pd
import dios
-from test.common import TESTFLAGGER
-
-
from saqc.funcs.tools import mask
from saqc.funcs.residues import calculatePolynomialResidues, calculateRollingResidues
+from test.fixtures import *
+from test.common import TESTFLAGGER
+
TF = TESTFLAGGER[:1]
@@ -45,8 +44,8 @@ def test_modelling_rollingMean_forRegular(dat, flagger):
data, _ = dat(freq="10min", periods=30, initial_level=0, final_level=100, out_val=-100)
data = dios.DictOfSeries(data)
flagger = flagger.initFlags(data)
- calculateRollingResidues(data, "data", flagger, 5, func=np.mean, eval_flags=True, min_periods=0, center=True)
- calculateRollingResidues(data, "data", flagger, 5, func=np.mean, eval_flags=True, min_periods=0, center=False)
+ calculateRollingResidues(data, "data", flagger, 5, func=np.mean, eval_flags=True, min_periods=0, center=True)
+ calculateRollingResidues(data, "data", flagger, 5, func=np.mean, eval_flags=True, min_periods=0, center=False)
@pytest.mark.parametrize("flagger", TF)
@@ -74,4 +73,4 @@ def test_modelling_mask(dat, flagger):
flagger = flagger.initFlags(data)
data_masked, flagger_masked = mask(data, "data", flagger, mode='mask_var', mask_var="mask_ser")
flaggs = flagger_masked._flags["data"]
- assert flaggs[data_masked['mask_ser']].isna().all()
\ No newline at end of file
+ assert flaggs[data_masked['mask_ser']].isna().all()
diff --git a/test/funcs/test_pattern_rec.py b/test/funcs/test_pattern_rec.py
index 0763a82f0..6f437edc2 100644
--- a/test/funcs/test_pattern_rec.py
+++ b/test/funcs/test_pattern_rec.py
@@ -2,10 +2,8 @@
# -*- coding: utf-8 -*-
import pytest
-
import pandas as pd
-
-from dios import dios
+import dios
from saqc.common import *
from saqc.flagger import Flagger, initFlagsLike
diff --git a/test/funcs/test_proc_functions.py b/test/funcs/test_proc_functions.py
index a7c99fc77..7f07e1d2e 100644
--- a/test/funcs/test_proc_functions.py
+++ b/test/funcs/test_proc_functions.py
@@ -2,22 +2,21 @@
# -*- coding: utf-8 -*-
-# see test/functs/conftest.py for global fixtures "course_..."
+# see test/functs/fixtures.py for global fixtures "course_..."
import pytest
import numpy as np
import pandas as pd
import dios
-from saqc.funcs.transformation import (
- transform
-)
+from saqc.common import *
+from saqc.funcs.transformation import transform
from saqc.funcs.drift import correctOffset
from saqc.funcs.interpolation import interpolateByRolling, interpolateInvalid, interpolateIndex
from saqc.funcs.resampling import resample
from saqc.lib.ts_operators import linearInterpolation, polynomialInterpolation
-from saqc.common import *
+from test.fixtures import *
from test.common import TESTFLAGGER
diff --git a/test/funcs/test_spikes_detection.py b/test/funcs/test_spikes_detection.py
index be38370e3..13526b62d 100644
--- a/test/funcs/test_spikes_detection.py
+++ b/test/funcs/test_spikes_detection.py
@@ -1,11 +1,12 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-# see test/functs/conftest.py for global fixtures "course_..."
+# see test/functs/fixtures.py for global fixtures "course_..."
import pytest
import numpy as np
import pandas as pd
import dios
+from test.fixtures import *
from saqc.funcs.outliers import (
flagMAD,
@@ -14,8 +15,6 @@ from saqc.funcs.outliers import (
flagMVScores,
flagByGrubbs,
)
-
-from test.common import TESTFLAGGER
from saqc.common import *
from saqc.flagger import Flagger, initFlagsLike
@@ -50,7 +49,7 @@ def test_flagSpikesBasic(spiky_data):
assert test_sum == len(spiky_data[1])
-# see test/functs/conftest.py for the 'course_N'
+# see test/functs/fixtures.py for the 'course_N'
@pytest.mark.parametrize(
"dat",
[
@@ -73,7 +72,7 @@ def test_flagSpikesLimitRaise(dat):
assert not np.any(flagger_result[field][characteristics["drop"]] > UNFLAGGED)
-# see test/functs/conftest.py for the 'course_N'
+# see test/functs/fixtures.py for the 'course_N'
@pytest.mark.parametrize("dat", [pytest.lazy_fixture("course_3")])
def test_flagMultivarScores(dat):
data1, characteristics = dat(periods=1000, initial_level=5, final_level=15, out_val=50)
diff --git a/test/run_pytest.py b/test/run_pytest.py
deleted file mode 100644
index 861cb7cc8..000000000
--- a/test/run_pytest.py
+++ /dev/null
@@ -1,4 +0,0 @@
-import pytest
-
-if __name__ == "__main__":
- pytest.main()
--
GitLab
From b12dd9a9abf9ee77f07b64d2f9db2e3b67d6e4a9 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 12 Mar 2021 15:50:49 +0100
Subject: [PATCH 042/180] added warning
---
saqc/funcs/flagtools.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index fb9522324..c9227a7d4 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -75,6 +75,10 @@ def clearFlags(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs
forceFlags : set whole column to a flag value
flagUnflagged : set flag value at all unflagged positions
"""
+ if 'flag' in kwargs:
+ flag = kwargs.pop('flag')
+ warnings.warn(f'`flag={flag}` is ignored here.')
+
return forceFlags(data, field, flagger, flag=UNFLAGGED, **kwargs)
--
GitLab
From 1b6099d28436630193f0fe6bfe3635e1f28c9e09 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 12 Mar 2021 15:56:13 +0100
Subject: [PATCH 043/180] changed structure of tests
---
test/flagger/__init__.py | 0
{testsfuzzy => tests}/__init__.py | 0
{test => tests}/common.py | 0
{test => tests}/core/__init__.py | 0
{test => tests}/core/test_core.py | 4 ++--
{test => tests}/core/test_creation.py | 0
{test => tests}/core/test_reader.py | 2 +-
{test => tests}/fixtures.py | 0
{test => tests/flagger}/__init__.py | 0
{test => tests}/flagger/test_flagger.py | 2 +-
{test => tests}/flagger/test_flags.py | 2 +-
{test => tests}/flagger/test_history.py | 0
{test => tests}/funcs/__init__.py | 0
{test => tests}/funcs/test_constants_detection.py | 2 +-
{test => tests}/funcs/test_functions.py | 9 +++------
{test => tests}/funcs/test_generic_api_functions.py | 4 +---
{test => tests}/funcs/test_generic_config_functions.py | 2 +-
{test => tests}/funcs/test_harm_funcs.py | 2 +-
{test => tests}/funcs/test_modelling.py | 7 ++-----
{test => tests}/funcs/test_pattern_rec.py | 4 ++--
{test => tests}/funcs/test_proc_functions.py | 7 ++-----
{test => tests}/funcs/test_spikes_detection.py | 7 ++-----
tests/fuzzy/__init__.py | 1 +
{testsfuzzy => tests/fuzzy}/init.py | 0
{testsfuzzy => tests/fuzzy}/test_functions.py | 2 +-
{testsfuzzy => tests/fuzzy}/test_masking.py | 9 ++-------
tests/lib/__init__.py | 1 +
{test => tests}/lib/test_rolling.py | 0
28 files changed, 25 insertions(+), 42 deletions(-)
delete mode 100644 test/flagger/__init__.py
rename {testsfuzzy => tests}/__init__.py (100%)
rename {test => tests}/common.py (100%)
rename {test => tests}/core/__init__.py (100%)
rename {test => tests}/core/test_core.py (97%)
rename {test => tests}/core/test_creation.py (100%)
rename {test => tests}/core/test_reader.py (98%)
rename {test => tests}/fixtures.py (100%)
rename {test => tests/flagger}/__init__.py (100%)
rename {test => tests}/flagger/test_flagger.py (99%)
rename {test => tests}/flagger/test_flags.py (99%)
rename {test => tests}/flagger/test_history.py (100%)
rename {test => tests}/funcs/__init__.py (100%)
rename {test => tests}/funcs/test_constants_detection.py (96%)
rename {test => tests}/funcs/test_functions.py (98%)
rename {test => tests}/funcs/test_generic_api_functions.py (95%)
rename {test => tests}/funcs/test_generic_config_functions.py (99%)
rename {test => tests}/funcs/test_harm_funcs.py (99%)
rename {test => tests}/funcs/test_modelling.py (96%)
rename {test => tests}/funcs/test_pattern_rec.py (94%)
rename {test => tests}/funcs/test_proc_functions.py (97%)
rename {test => tests}/funcs/test_spikes_detection.py (96%)
create mode 100644 tests/fuzzy/__init__.py
rename {testsfuzzy => tests/fuzzy}/init.py (100%)
rename {testsfuzzy => tests/fuzzy}/test_functions.py (97%)
rename {testsfuzzy => tests/fuzzy}/test_masking.py (96%)
create mode 100644 tests/lib/__init__.py
rename {test => tests}/lib/test_rolling.py (100%)
diff --git a/test/flagger/__init__.py b/test/flagger/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/testsfuzzy/__init__.py b/tests/__init__.py
similarity index 100%
rename from testsfuzzy/__init__.py
rename to tests/__init__.py
diff --git a/test/common.py b/tests/common.py
similarity index 100%
rename from test/common.py
rename to tests/common.py
diff --git a/test/core/__init__.py b/tests/core/__init__.py
similarity index 100%
rename from test/core/__init__.py
rename to tests/core/__init__.py
diff --git a/test/core/test_core.py b/tests/core/test_core.py
similarity index 97%
rename from test/core/test_core.py
rename to tests/core/test_core.py
index 51b99f5c4..cddc2fd59 100644
--- a/test/core/test_core.py
+++ b/tests/core/test_core.py
@@ -7,12 +7,12 @@ import numpy as np
import pandas as pd
from saqc.common import *
-from saqc.flagger import Flagger, initFlagsLike
+from saqc.flagger import initFlagsLike
from saqc.funcs import flagRange
from saqc.lib import plotting as splot
from saqc import SaQC, register
-from test.common import initData, flagAll
+from tests.common import initData, flagAll
# no logging output needed here
# -> can this be configured on the test runner level?
diff --git a/test/core/test_creation.py b/tests/core/test_creation.py
similarity index 100%
rename from test/core/test_creation.py
rename to tests/core/test_creation.py
diff --git a/test/core/test_reader.py b/tests/core/test_reader.py
similarity index 98%
rename from test/core/test_reader.py
rename to tests/core/test_reader.py
index 9ab7d2a51..e2d80042b 100644
--- a/test/core/test_reader.py
+++ b/tests/core/test_reader.py
@@ -11,7 +11,7 @@ from saqc.core.config import Fields as F
from saqc.core.core import SaQC
from saqc.core.register import FUNC_MAP, register
-from test.common import initData, writeIO
+from tests.common import initData, writeIO
@pytest.fixture
diff --git a/test/fixtures.py b/tests/fixtures.py
similarity index 100%
rename from test/fixtures.py
rename to tests/fixtures.py
diff --git a/test/__init__.py b/tests/flagger/__init__.py
similarity index 100%
rename from test/__init__.py
rename to tests/flagger/__init__.py
diff --git a/test/flagger/test_flagger.py b/tests/flagger/test_flagger.py
similarity index 99%
rename from test/flagger/test_flagger.py
rename to tests/flagger/test_flagger.py
index a304409fb..1af9f4710 100644
--- a/test/flagger/test_flagger.py
+++ b/tests/flagger/test_flagger.py
@@ -7,7 +7,7 @@ from pandas.api.types import is_bool_dtype
import dios
-from test.common import TESTFLAGGER, initData
+from tests.common import TESTFLAGGER, initData
pytestmark = pytest.mark.skip('old flagger tests - rewrite needed')
diff --git a/test/flagger/test_flags.py b/tests/flagger/test_flags.py
similarity index 99%
rename from test/flagger/test_flags.py
rename to tests/flagger/test_flags.py
index c04177101..652022048 100644
--- a/test/flagger/test_flags.py
+++ b/tests/flagger/test_flags.py
@@ -7,7 +7,7 @@ import pandas as pd
from saqc.common import *
from saqc.flagger.flags import Flags
-from test.flagger.test_history import (
+from tests.flagger.test_history import (
History,
is_equal as hist_equal,
)
diff --git a/test/flagger/test_history.py b/tests/flagger/test_history.py
similarity index 100%
rename from test/flagger/test_history.py
rename to tests/flagger/test_history.py
diff --git a/test/funcs/__init__.py b/tests/funcs/__init__.py
similarity index 100%
rename from test/funcs/__init__.py
rename to tests/funcs/__init__.py
diff --git a/test/funcs/test_constants_detection.py b/tests/funcs/test_constants_detection.py
similarity index 96%
rename from test/funcs/test_constants_detection.py
rename to tests/funcs/test_constants_detection.py
index 30a16db10..1ae7be198 100644
--- a/test/funcs/test_constants_detection.py
+++ b/tests/funcs/test_constants_detection.py
@@ -8,7 +8,7 @@ from saqc.common import *
from saqc.funcs.constants import flagConstants, flagByVariance
from saqc.flagger import initFlagsLike
-from test.common import initData
+from tests.common import initData
@pytest.fixture
diff --git a/test/funcs/test_functions.py b/tests/funcs/test_functions.py
similarity index 98%
rename from test/funcs/test_functions.py
rename to tests/funcs/test_functions.py
index a205e517d..bbaa20ad5 100644
--- a/test/funcs/test_functions.py
+++ b/tests/funcs/test_functions.py
@@ -1,13 +1,10 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-import pytest
-import pandas as pd
-import numpy as np
import dios
from saqc.common import *
-from saqc.flagger import Flagger, initFlagsLike
+from saqc.flagger import initFlagsLike
from saqc.funcs.drift import flagDriftFromNorm, flagDriftFromReference, flagDriftFromScaledNorm
from saqc.funcs.outliers import flagCrossStatistic, flagRange
from saqc.funcs.flagtools import flagManual, forceFlags, clearFlags
@@ -15,8 +12,8 @@ from saqc.funcs.tools import drop, copy, mask
from saqc.funcs.resampling import reindexFlags
from saqc.funcs.breaks import flagIsolated
-from test.fixtures import *
-from test.common import initData
+from tests.fixtures import *
+from tests.common import initData
@pytest.fixture
diff --git a/test/funcs/test_generic_api_functions.py b/tests/funcs/test_generic_api_functions.py
similarity index 95%
rename from test/funcs/test_generic_api_functions.py
rename to tests/funcs/test_generic_api_functions.py
index 76c9f0d53..d581c4344 100644
--- a/test/funcs/test_generic_api_functions.py
+++ b/tests/funcs/test_generic_api_functions.py
@@ -1,9 +1,7 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-import ast
import pytest
-import numpy as np
import pandas as pd
from saqc.common import *
@@ -11,7 +9,7 @@ from saqc.core.register import register
from saqc.funcs.tools import mask
from saqc import SaQC
-from test.common import initData, flagAll
+from tests.common import initData, flagAll
register(masking='field')(flagAll)
diff --git a/test/funcs/test_generic_config_functions.py b/tests/funcs/test_generic_config_functions.py
similarity index 99%
rename from test/funcs/test_generic_config_functions.py
rename to tests/funcs/test_generic_config_functions.py
index 81e91d643..6e84e6a99 100644
--- a/test/funcs/test_generic_config_functions.py
+++ b/tests/funcs/test_generic_config_functions.py
@@ -15,7 +15,7 @@ from saqc.core.register import register
from saqc.funcs.generic import _execGeneric
from saqc import SaQC
-from test.common import TESTNODATA, initData, writeIO
+from tests.common import TESTNODATA, initData, writeIO
@pytest.fixture
diff --git a/test/funcs/test_harm_funcs.py b/tests/funcs/test_harm_funcs.py
similarity index 99%
rename from test/funcs/test_harm_funcs.py
rename to tests/funcs/test_harm_funcs.py
index 187a4b7c4..4f12a41d2 100644
--- a/test/funcs/test_harm_funcs.py
+++ b/tests/funcs/test_harm_funcs.py
@@ -8,7 +8,7 @@ import numpy as np
import pandas as pd
import dios
-from test.common import TESTFLAGGER
+from tests.common import TESTFLAGGER
from saqc.funcs.resampling import (
linear,
diff --git a/test/funcs/test_modelling.py b/tests/funcs/test_modelling.py
similarity index 96%
rename from test/funcs/test_modelling.py
rename to tests/funcs/test_modelling.py
index 748a06fa3..6d99d5786 100644
--- a/test/funcs/test_modelling.py
+++ b/tests/funcs/test_modelling.py
@@ -4,16 +4,13 @@
# see test/functs/fixtures.py for global fixtures "course_..."
-import pytest
-import numpy as np
-import pandas as pd
import dios
from saqc.funcs.tools import mask
from saqc.funcs.residues import calculatePolynomialResidues, calculateRollingResidues
-from test.fixtures import *
-from test.common import TESTFLAGGER
+from tests.fixtures import *
+from tests.common import TESTFLAGGER
TF = TESTFLAGGER[:1]
diff --git a/test/funcs/test_pattern_rec.py b/tests/funcs/test_pattern_rec.py
similarity index 94%
rename from test/funcs/test_pattern_rec.py
rename to tests/funcs/test_pattern_rec.py
index 6f437edc2..3ca69d707 100644
--- a/test/funcs/test_pattern_rec.py
+++ b/tests/funcs/test_pattern_rec.py
@@ -6,9 +6,9 @@ import pandas as pd
import dios
from saqc.common import *
-from saqc.flagger import Flagger, initFlagsLike
+from saqc.flagger import initFlagsLike
from saqc.funcs.pattern import *
-from test.common import initData
+from tests.common import initData
@pytest.fixture
diff --git a/test/funcs/test_proc_functions.py b/tests/funcs/test_proc_functions.py
similarity index 97%
rename from test/funcs/test_proc_functions.py
rename to tests/funcs/test_proc_functions.py
index 7f07e1d2e..31337badc 100644
--- a/test/funcs/test_proc_functions.py
+++ b/tests/funcs/test_proc_functions.py
@@ -4,9 +4,6 @@
# see test/functs/fixtures.py for global fixtures "course_..."
-import pytest
-import numpy as np
-import pandas as pd
import dios
from saqc.common import *
@@ -16,8 +13,8 @@ from saqc.funcs.interpolation import interpolateByRolling, interpolateInvalid, i
from saqc.funcs.resampling import resample
from saqc.lib.ts_operators import linearInterpolation, polynomialInterpolation
-from test.fixtures import *
-from test.common import TESTFLAGGER
+from tests.fixtures import *
+from tests.common import TESTFLAGGER
@pytest.mark.parametrize("flagger", TESTFLAGGER)
diff --git a/test/funcs/test_spikes_detection.py b/tests/funcs/test_spikes_detection.py
similarity index 96%
rename from test/funcs/test_spikes_detection.py
rename to tests/funcs/test_spikes_detection.py
index 13526b62d..578dd9c44 100644
--- a/test/funcs/test_spikes_detection.py
+++ b/tests/funcs/test_spikes_detection.py
@@ -2,11 +2,8 @@
# -*- coding: utf-8 -*-
# see test/functs/fixtures.py for global fixtures "course_..."
-import pytest
-import numpy as np
-import pandas as pd
import dios
-from test.fixtures import *
+from tests.fixtures import *
from saqc.funcs.outliers import (
flagMAD,
@@ -16,7 +13,7 @@ from saqc.funcs.outliers import (
flagByGrubbs,
)
from saqc.common import *
-from saqc.flagger import Flagger, initFlagsLike
+from saqc.flagger import initFlagsLike
@pytest.fixture(scope="module")
diff --git a/tests/fuzzy/__init__.py b/tests/fuzzy/__init__.py
new file mode 100644
index 000000000..4265cc3e6
--- /dev/null
+++ b/tests/fuzzy/__init__.py
@@ -0,0 +1 @@
+#!/usr/bin/env python
diff --git a/testsfuzzy/init.py b/tests/fuzzy/init.py
similarity index 100%
rename from testsfuzzy/init.py
rename to tests/fuzzy/init.py
diff --git a/testsfuzzy/test_functions.py b/tests/fuzzy/test_functions.py
similarity index 97%
rename from testsfuzzy/test_functions.py
rename to tests/fuzzy/test_functions.py
index fc3caa00e..09d1f8484 100644
--- a/testsfuzzy/test_functions.py
+++ b/tests/fuzzy/test_functions.py
@@ -6,7 +6,7 @@ from hypothesis import given, settings
from hypothesis.strategies import data, from_type
from saqc.core.register import FUNC_MAP
-from testsfuzzy.init import MAX_EXAMPLES, functionKwargs
+from tests.fuzzy.init import MAX_EXAMPLES, functionKwargs
@settings(max_examples=MAX_EXAMPLES, deadline=None)
diff --git a/testsfuzzy/test_masking.py b/tests/fuzzy/test_masking.py
similarity index 96%
rename from testsfuzzy/test_masking.py
rename to tests/fuzzy/test_masking.py
index b1eb5861e..9d45520eb 100644
--- a/testsfuzzy/test_masking.py
+++ b/tests/fuzzy/test_masking.py
@@ -6,17 +6,12 @@ import logging
import pandas as pd
from hypothesis import given, settings
-from hypothesis.strategies import (
- sampled_from,
- composite,
- sampled_from,
-)
from saqc.common import *
-from saqc.flagger import Flagger, initFlagsLike
+from saqc.flagger import Flagger
from saqc.core.register import _maskData, _unmaskData
-from testsfuzzy.init import dataFieldFlagger, MAX_EXAMPLES
+from tests.fuzzy.init import dataFieldFlagger, MAX_EXAMPLES
logging.disable(logging.CRITICAL)
diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py
new file mode 100644
index 000000000..4265cc3e6
--- /dev/null
+++ b/tests/lib/__init__.py
@@ -0,0 +1 @@
+#!/usr/bin/env python
diff --git a/test/lib/test_rolling.py b/tests/lib/test_rolling.py
similarity index 100%
rename from test/lib/test_rolling.py
rename to tests/lib/test_rolling.py
--
GitLab
From ffec93924029ba25d2af4d6c40ed895517d6c930 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 12 Mar 2021 16:16:55 +0100
Subject: [PATCH 044/180] new CI
---
.gitlab-ci.yml | 79 ++++++++++++++++++++++++++++++++------------------
1 file changed, 50 insertions(+), 29 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c014ef029..6de191455 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,46 +1,67 @@
variables:
GIT_SUBMODULE_STRATEGY: recursive
+
+default:
+ image: python:3.8
+
+
before_script:
- - export DEBIAN_FRONTEND=noninteractive
- - apt-get -qq update
- - apt-get -qq install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl git > /dev/null
- - export DEBIAN_FRONTEND=dialog
- - export LC_ALL=C.UTF-8
- - export LANG=C.UTF-8
- - git clone https://github.com/pyenv/pyenv.git ~/.pyenv
- - export PYENV_ROOT="$HOME/.pyenv"
- - export PATH="$PYENV_ROOT/bin:$PATH"
- - eval "$(pyenv init -)"
-
-
-test:python37:
+ - pip install --upgrade pip
+ - pip install pytest
+ - pip install -r requirements.txt
+
+
+# test saqc with python 3.7
+python37:
+ stage: test
+ image: python:3.7
script:
- - pyenv install 3.7.5
- - pyenv shell 3.7.5
- - pip install --upgrade pip
- - pip install -r requirements.txt
- - python -m pytest --ignore test/lib test
+ - pytest tests/core tests/flagger tests/funcs
- python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv --outfile /tmp/test.csv
-test:python38:
+# test saqc with python 3.8
+python38:
+ stage: test
script:
- - pyenv install 3.8.0
- - pyenv shell 3.8.0
- - pip install --upgrade pip
- - pip install -r requirements.txt
- - python -m pytest --ignore test/lib test
+ - pytest tests/core tests/flagger tests/funcs
- python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv --outfile /tmp/test.csv
-# Make html docu with sphinx
+
+# test lib saqc
+testLib:
+ stage: test
+ script:
+ - pytest tests/lib
+
+
+# fuzzy testing saqc
+fuzzy:
+ stage: test
+ script:
+ - pytest tests/fuzzy
+ allow_failure: true
+
+
+# make (visual) coverage in gitlab merge request diff's
+coverage:
+ stage: test
+ script:
+ - pip install pytest-cov coverage
+ - pytest --cov=saqc tests/core tests/flagger tests/funcs
+ - coverage xml
+ # regex to find the coverage percentage in the job output
+ coverage: '/^TOTAL.+?(\d+\%)$/'
+ artifacts:
+ reports:
+ cobertura: coverage.xml
+
+
+# make html docu with sphinx
pages:
stage: deploy
script:
- - pyenv install 3.8.0
- - pyenv shell 3.8.0
- - pip install --upgrade pip
- - pip install -r requirements.txt
- cd sphinx-doc/
- pip install -r requirements_sphinx.txt
- make doc
--
GitLab
From 24e100c0c752bd88afb045e7ce4d6cda52bdc014 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 12 Mar 2021 16:23:33 +0100
Subject: [PATCH 045/180] new CI
---
.gitlab-ci.yml | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6de191455..eb0616e66 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -38,22 +38,28 @@ testLib:
# fuzzy testing saqc
fuzzy:
+ allow_failure: true
stage: test
script:
- pytest tests/fuzzy
- allow_failure: true
# make (visual) coverage in gitlab merge request diff's
coverage:
+ allow_failure: true
stage: test
+
script:
- pip install pytest-cov coverage
- pytest --cov=saqc tests/core tests/flagger tests/funcs
+ after_script:
- coverage xml
+
# regex to find the coverage percentage in the job output
coverage: '/^TOTAL.+?(\d+\%)$/'
+
artifacts:
+ when: always
reports:
cobertura: coverage.xml
--
GitLab
From 66f9a340fe739e3602f97042b48baaeb0252dd59 Mon Sep 17 00:00:00 2001
From: Peter Luenenschloss <peter.luenenschloss@ufz.de>
Date: Mon, 15 Mar 2021 20:25:12 +0100
Subject: [PATCH 046/180] interpolation rework
---
saqc/funcs/interpolation.py | 168 +++++++++++-------------------------
saqc/lib/ts_operators.py | 20 +----
2 files changed, 53 insertions(+), 135 deletions(-)
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index a615aee92..73fdd3eef 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -12,6 +12,7 @@ from dios import DictOfSeries
from saqc.common import *
from saqc.core.register import register
from saqc.flagger import Flagger
+from saqc.flagger.flags import applyFunctionOnHistory
from saqc.lib.tools import toSequence, evalFreqStr, getDropMask
from saqc.lib.ts_operators import interpolateNANs
@@ -166,20 +167,30 @@ def interpolateInvalid(
return data, flagger
-@register(masking='field', module="interpolation")
+def _overlap_rs(x, freq='1min', fill_value=-np.inf):
+ x = x.resample(freq).max()
+ x = x.combine(x.shift(1, fill_value=fill_value), max)
+ # we are appending last regular grid entry (if necessary), to conserve integrity of groups of regularized
+ # timestamps originating all from the same logger.
+ try:
+ x = x.append(pd.Series([-np.inf], index=[x.index[-1].ceil(freq)]), verify_integrity=True)
+ except ValueError:
+ pass
+ return x
+
+
+@register(masking='none', module="interpolation")
def interpolateIndex(
data: DictOfSeries,
field: str,
flagger: Flagger,
freq: str,
- method: Literal["linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric", "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"],
+ method: Literal["linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric",
+ "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"],
inter_order: int=2,
- to_drop: Optional[Union[Any, Sequence[Any]]]=None,
downgrade_interpolation: bool=False,
- empty_intervals_flag: Any=None,
- grid_field: str=None,
inter_limit: int=2,
- freq_check: Optional[Literal["check", "auto"]]=None, # TODO: rm not a user decision
+ to_mask: Optional[Union[Any, Sequence[Any]]]=BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
@@ -197,6 +208,8 @@ def interpolateIndex(
you want to interpolate, to "grid_field". 'freq' is then use to determine the maximum gap size for
a grid point to be interpolated.
+ Note, that intervals, not having an interpolation value assigned (thus, evaluate to np.nan), get UNFLAGGED assigned.
+
Parameters
----------
data : dios.DictOfSeries
@@ -214,29 +227,13 @@ def interpolateIndex(
inter_order : integer, default 2
If there your selected interpolation method can be performed at different 'orders' - here you pass the desired
order.
- to_drop : {None, str, List[str]}, default None
- Flags that refer to values you want to drop before interpolation - effectively excluding grid points from
- interpolation, that are only surrounded by values having a flag in them, that is listed in drop flags. Default
- results in the flaggers *BAD* flag to be the drop_flag.
downgrade_interpolation : bool, default False
If interpolation can not be performed at `inter_order` - (not enough values or not implemented at this order) -
automatically try to interpolate at order `inter_order` :math:`- 1`.
- empty_intervals_flag : str, default None
- A Flag, that you want to assign to those values in the resulting equidistant sample grid, that were not
- surrounded by valid data in the original dataset, and thus were not interpolated. Default automatically assigns
- ``BAD`` flag to those values.
- grid_field : String, default None
- Use the timestamp of another variable as (not necessarily regular) "grid" to be interpolated.
inter_limit : Integer, default 2
Maximum number of consecutive Grid values allowed for interpolation. If set
to *n*, chunks of *n* and more consecutive grid values, where there is no value in between, wont be
interpolated.
- freq_check : {None, 'check', 'auto'}, default None
-
- * ``None``: do not validate frequency-string passed to `freq`
- * ``'check'``: estimate frequency and log a warning if estimate miss matchs frequency string passed to 'freq', or
- if no uniform sampling rate could be estimated
- * ``'auto'``: estimate frequency and use estimate. (Ignores `freq` parameter.)
Returns
-------
@@ -247,123 +244,58 @@ def interpolateIndex(
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
- raise NotImplementedError("currently not available - rewrite needed")
- datcol = data[field]
- datcol = datcol.copy()
- flagscol = flagger.getFlags(field)
- freq = evalFreqStr(freq, freq_check, datcol.index)
+ if to_mask is None:
+ to_mask = BAD
- if empty_intervals_flag is None:
- empty_intervals_flag = BAD
+ datcol = data[field]
+ if datcol.empty:
+ return data, flagger
- drop_mask = getDropMask(field, to_drop, flagger, BAD)
- drop_mask |= flagscol.isna()
- drop_mask |= datcol.isna()
- datcol[drop_mask] = np.nan
+ start, end = datcol.index[0].floor(freq), datcol.index[-1].ceil(freq)
+ datcol = datcol.copy()
datcol.dropna(inplace=True)
- if datcol.empty:
- data[field] = datcol
- reshaped_flagger = flagger.initFlags(datcol).setFlags(field, flag=flagscol, force=True, inplace=True, **kwargs)
- flagger = flagger.slice(drop=field).merge(reshaped_flagger, subset=[field], inplace=True)
- return data, flagger
+ grid_index = pd.date_range(start=start, end=end, freq=freq, name=datcol.index.name)
+ dat_index = datcol.index
# account for annoying case of subsequent frequency aligned values,
# which differ exactly by the margin of 2*freq
- spec_case_mask = datcol.index.to_series()
- spec_case_mask = spec_case_mask - spec_case_mask.shift(1)
- spec_case_mask = spec_case_mask == 2 * pd.Timedelta(freq)
- spec_case_mask = spec_case_mask[spec_case_mask]
- spec_case_mask = spec_case_mask.resample(freq).asfreq().dropna()
-
- if not spec_case_mask.empty:
- spec_case_mask = spec_case_mask.tshift(-1, freq)
+ gaps = ((dat_index[1:] - dat_index[:-1]) == 2*pd.Timedelta(freq))
+ gaps = dat_index[1:][gaps]
+ aligned_gaps = gaps.join(grid_index, how='inner')
+ if not aligned_gaps.empty:
+ aligned_gaps = aligned_gaps.shift(-1, freq)
# prepare grid interpolation:
- if grid_field is None:
- start, end = datcol.index[0].floor(freq), datcol.index[-1].ceil(freq)
- grid_index = pd.date_range(start=start, end=end, freq=freq, name=datcol.index.name)
- else:
- grid_index = data[grid_field].index
-
- aligned_start = datcol.index[0] == grid_index[0]
- aligned_end = datcol.index[-1] == grid_index[-1]
datcol = datcol.reindex(datcol.index.join(grid_index, how="outer",))
- # do the interpolation
- inter_data, chunk_bounds = interpolateNANs(
+ # do the grid interpolation
+ inter_data = interpolateNANs(
data=datcol,
method=method,
order=inter_order,
inter_limit=inter_limit,
downgrade_interpolation=downgrade_interpolation,
- return_chunk_bounds=True
)
# override falsely interpolated values:
- if grid_field is None:
- inter_data[spec_case_mask.index] = np.nan
+ inter_data[aligned_gaps] = np.nan
# store interpolated grid
- inter_data = inter_data[grid_index]
- data[field] = inter_data
-
- # flags reshaping (dropping data drops):
- flagscol.drop(flagscol[drop_mask].index, inplace=True)
-
- if grid_field is not None:
- # only basic flag propagation supported for custom grids (take worst from preceeding/succeeding)
- preceeding = flagscol.reindex(grid_index, method='ffill', tolerance=freq)
- succeeding = flagscol.reindex(grid_index, method='bfill', tolerance=freq)
- # check for too big gaps in the source data and drop the values interpolated in those too big gaps
- na_mask = preceeding.isna() | succeeding.isna()
- na_mask = na_mask[na_mask]
- preceeding.drop(na_mask.index, inplace=True)
- succeeding.drop(na_mask.index, inplace=True)
- inter_data.drop(na_mask.index, inplace=True)
- data[field] = inter_data
- mask = succeeding > preceeding
- preceeding.loc[mask] = succeeding.loc[mask]
- flagscol = preceeding
- flagger_new = flagger.initFlags(inter_data).setFlags(field, flag=flagscol, force=True, **kwargs)
- flagger = flagger.slice(drop=field).merge(flagger_new)
- return data, flagger
+ data[field] = inter_data[grid_index]
+
+ # flags reshaping
+ flagscol = flagger[field]
+ flagscol.drop(flagscol[flagscol >= to_mask].index, inplace=True)
+
+ flagscol = _overlap_rs(flagscol, freq, UNFLAGGED)
+ flagger = applyFunctionOnHistory(
+ flagger,
+ field,
+ hist_func=_overlap_rs, hist_kws=dict(fill_value=UNFLAGGED),
+ mask_func=_overlap_rs, mask_kws=dict(fill_value=False),
+ last_column=flagscol
+ )
- # for freq defined grids, max-aggregate flags of every grid points freq-ranged surrounding
- # hack ahead! Resampling with overlapping intervals:
- # 1. -> no rolling over categories allowed in pandas, so we translate manually:
- cats = pd.CategoricalIndex(flagger.dtype.categories, ordered=True)
- cats_dict = {cats[i]: i for i in range(0, len(cats))}
- flagscol = flagscol.replace(cats_dict)
- # 3. -> combine resample+rolling to resample with overlapping intervals:
- flagscol = flagscol.resample(freq).max()
- initial = flagscol[0]
- flagscol = flagscol.rolling(2, center=True, closed="neither").max()
- flagscol[0] = initial
- cats_dict = {num: key for (key, num) in cats_dict.items()}
- flagscol = flagscol.astype(int, errors="ignore").replace(cats_dict)
- flagscol[flagscol.isna()] = empty_intervals_flag
-
- # we might miss the flag for interpolated data grids last entry (if we miss it - the datapoint is always nan
- # - just settling a convention here(resulting GRID should start BEFORE first valid data entry and range to AFTER
- # last valid data)):
- if inter_data.shape[0] > flagscol.shape[0]:
- flagscol = flagscol.append(pd.Series(empty_intervals_flag, index=[datcol.index[-1]]))
-
- # Additional consistency operation: we have to block first/last interpolated datas flags - since they very
- # likely represent chunk starts/ends (except data start and or end timestamp were grid-aligned before Grid
- # interpolation already.)
- if np.isnan(inter_data[0]) and not aligned_start:
- chunk_bounds = chunk_bounds.insert(0, inter_data.index[0])
- if np.isnan(inter_data[-1]) and not aligned_end:
- chunk_bounds = chunk_bounds.append(pd.DatetimeIndex([inter_data.index[-1]]))
- chunk_bounds = chunk_bounds.unique()
- flagger_new = flagger.initFlags(inter_data).setFlags(field, flag=flagscol, force=True, inplace=True, **kwargs)
-
- # block chunk ends of interpolation
- flags_to_block = pd.Series(np.nan, index=chunk_bounds).astype(flagger_new.dtype)
- flagger_new = flagger_new.setFlags(field, loc=chunk_bounds, flag=flags_to_block, force=True, inplace=True)
-
- flagger = flagger.slice(drop=field).merge(flagger_new, subset=[field], inplace=True)
return data, flagger
diff --git a/saqc/lib/ts_operators.py b/saqc/lib/ts_operators.py
index 8f1f86cef..44f91cb64 100644
--- a/saqc/lib/ts_operators.py
+++ b/saqc/lib/ts_operators.py
@@ -179,7 +179,7 @@ def meanQC(data, max_nan_total=np.inf, max_nan_consec=np.inf):
return np.nanmean(data[~validationTrafo(data.isna(), max_nan_total, max_nan_consec)])
-def interpolateNANs(data, method, order=2, inter_limit=2, downgrade_interpolation=False, return_chunk_bounds=False):
+def interpolateNANs(data, method, order=2, inter_limit=2, downgrade_interpolation=False):
"""
The function interpolates nan-values (and nan-grids) in timeseries data. It can be passed all the method keywords
from the pd.Series.interpolate method and will than apply this very methods. Note, that the inter_limit keyword
@@ -198,12 +198,6 @@ def interpolateNANs(data, method, order=2, inter_limit=2, downgrade_interpolatio
:param downgrade_interpolation: Boolean. Default False. If True:
If a data chunk not contains enough values for interpolation of the order "order",
the highest order possible will be selected for that chunks interpolation.
- :param return_chunk_bounds: Boolean. Default False. If True:
- Additionally to the interpolated data, the start and ending points of data chunks
- not containing no series consisting of more then "inter_limit" nan values,
- are calculated and returned.
- (This option fits requirements of the "interpolateNANs" functions use in the
- context of saqc harmonization mainly.)
:return:
"""
@@ -218,13 +212,6 @@ def interpolateNANs(data, method, order=2, inter_limit=2, downgrade_interpolatio
gap_mask.replace(True, np.nan).fillna(method="bfill", limit=inter_limit).replace(np.nan, True).astype(bool)
)
- if return_chunk_bounds:
- # start end ending points of interpolation chunks have to be memorized to block their flagging:
- chunk_switches = gap_mask.astype(int).diff()
- chunk_starts = chunk_switches[chunk_switches == -1].index
- chunk_ends = chunk_switches[(chunk_switches.shift(-1) == 1)].index
- chunk_bounds = chunk_starts.join(chunk_ends, how="outer", sort=True)
-
pre_index = data.index
data = data[gap_mask]
@@ -260,9 +247,8 @@ def interpolateNANs(data, method, order=2, inter_limit=2, downgrade_interpolatio
data = data.squeeze(axis=1)
data.name = dat_name
data = data.reindex(pre_index)
- if return_chunk_bounds:
- return data, chunk_bounds
- else: return data
+
+ return data
def aggregate2Freq(
--
GitLab
From 02dd7f3c467b5178a36e7039ab9085a9fba9e5a7 Mon Sep 17 00:00:00 2001
From: Peter Luenenschloss <peter.luenenschloss@ufz.de>
Date: Mon, 15 Mar 2021 21:19:58 +0100
Subject: [PATCH 047/180] tired
---
saqc/funcs/interpolation.py | 10 +++++-----
test/funcs/test_harm_funcs.py | 6 ++++--
2 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index 73fdd3eef..8efb6537e 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -151,8 +151,7 @@ def interpolateInvalid(
method,
order=inter_order,
inter_limit=inter_limit,
- downgrade_interpolation=downgrade_interpolation,
- return_chunk_bounds=False,
+ downgrade_interpolation=downgrade_interpolation
)
interpolated = data[field].isna() & inter_data.notna()
@@ -168,12 +167,13 @@ def interpolateInvalid(
def _overlap_rs(x, freq='1min', fill_value=-np.inf):
+ end = x.index[-1].ceil(freq)
x = x.resample(freq).max()
x = x.combine(x.shift(1, fill_value=fill_value), max)
# we are appending last regular grid entry (if necessary), to conserve integrity of groups of regularized
# timestamps originating all from the same logger.
try:
- x = x.append(pd.Series([-np.inf], index=[x.index[-1].ceil(freq)]), verify_integrity=True)
+ x = x.append(pd.Series([-np.inf], index=[end]), verify_integrity=True)
except ValueError:
pass
return x
@@ -293,8 +293,8 @@ def interpolateIndex(
flagger = applyFunctionOnHistory(
flagger,
field,
- hist_func=_overlap_rs, hist_kws=dict(fill_value=UNFLAGGED),
- mask_func=_overlap_rs, mask_kws=dict(fill_value=False),
+ hist_func=_overlap_rs, hist_kws=dict(freq=freq, fill_value=UNFLAGGED),
+ mask_func=_overlap_rs, mask_kws=dict(freq=freq, fill_value=False),
last_column=flagscol
)
diff --git a/test/funcs/test_harm_funcs.py b/test/funcs/test_harm_funcs.py
index 2fca105a6..41408611f 100644
--- a/test/funcs/test_harm_funcs.py
+++ b/test/funcs/test_harm_funcs.py
@@ -10,6 +10,8 @@ import pandas as pd
import dios
from test.common import TESTFLAGGER
+from saqc.flagger import Flagger, initFlagsLike
+from saqc.common import BAD
from saqc.funcs.resampling import (
linear,
@@ -43,10 +45,10 @@ def data():
@pytest.mark.parametrize("flagger", TESTFLAGGER)
@pytest.mark.parametrize("reshaper", RESHAPERS)
def test_harmSingleVarIntermediateFlagging(data, flagger, reshaper):
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
# make pre harm copies:
pre_data = data.copy()
- pre_flags = flagger.getFlags()
+ pre_flags = flagger['data']
freq = "15min"
assert len(data.columns) == 1
field = data.columns[0]
--
GitLab
From 1583f04708c53646d0db1c149741f250c9467d03 Mon Sep 17 00:00:00 2001
From: Peter Luenenschloss <peter.luenenschloss@ufz.de>
Date: Tue, 16 Mar 2021 07:00:10 +0100
Subject: [PATCH 048/180] fixed the nastiest of all my bugs
---
saqc/funcs/interpolation.py | 2 +-
saqc/funcs/resampling.py | 67 ++++++++++++++++++-------------------
2 files changed, 33 insertions(+), 36 deletions(-)
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index 8efb6537e..fecb5718d 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -173,7 +173,7 @@ def _overlap_rs(x, freq='1min', fill_value=-np.inf):
# we are appending last regular grid entry (if necessary), to conserve integrity of groups of regularized
# timestamps originating all from the same logger.
try:
- x = x.append(pd.Series([-np.inf], index=[end]), verify_integrity=True)
+ x = x.append(pd.Series([fill_value], index=[end]), verify_integrity=True)
except ValueError:
pass
return x
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 3a4d8ce78..b7fb72135 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -19,6 +19,7 @@ from saqc.funcs.interpolation import interpolateIndex
from saqc.lib.tools import getDropMask, evalFreqStr
from saqc.lib.ts_operators import shift2Freq, aggregate2Freq
from saqc.flagger.flags import applyFunctionOnHistory
+from saqc.lib.rolling import customRoller
logger = logging.getLogger("SaQC")
@@ -564,7 +565,9 @@ def resample(
# create a dummys
if all_na_2_empty and datcol.dropna().empty:
-
+ # Todo: This needs discussion. It makes possible, that different harmonized variables,
+ # resulting from the harmonization of the same logger, have differing timestamps!
+ # (Same holds for starting/ending nan-chunk truncation)
datcol = pd.Series([], index=pd.DatetimeIndex([]), name=field)
flagscol = pd.Series([], index=pd.DatetimeIndex([]), name=field)
@@ -614,7 +617,24 @@ def resample(
return data, flagger
-@register(masking='field', module="resampling")
+def _getChunkBounds(target_datcol, flagscol, freq):
+ chunk_end = target_datcol.reindex(flagscol.index, method='bfill', tolerance=freq)
+ chunk_start = target_datcol.reindex(flagscol.index, method='ffill', tolerance=freq)
+ ignore_flags = (chunk_end.isna() | chunk_start.isna())
+ return ignore_flags
+
+
+def _inverseInterpolation(target_flagscol, flagscol=None, freq=None):
+ backprojected = flagscol.reindex(target_flagscol.index, method="bfill", tolerance=freq)
+ fwrdprojected = flagscol.reindex(target_flagscol.index, method="ffill", tolerance=freq)
+ b_replacement_mask = (backprojected > target_flagscol) & (backprojected >= fwrdprojected)
+ f_replacement_mask = (fwrdprojected > target_flagscol) & (fwrdprojected > backprojected)
+ target_flagscol.loc[b_replacement_mask] = backprojected.loc[b_replacement_mask]
+ target_flagscol.loc[f_replacement_mask] = fwrdprojected.loc[f_replacement_mask]
+ return target_flagscol
+
+
+@register(masking='none', module="resampling")
def reindexFlags(
data: DictOfSeries,
field: str,
@@ -622,8 +642,7 @@ def reindexFlags(
method: Literal["inverse_fagg", "inverse_bagg", "inverse_nagg", "inverse_fshift", "inverse_bshift", "inverse_nshift"],
source: str,
freq: Optional[str]=None,
- to_drop: Optional[Union[Any, Sequence[Any]]]=None,
- freq_check: Optional[Literal["check", "auto"]]=None,
+ to_mask: Optional[Union[Any, Sequence[Any]]]=BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
@@ -674,14 +693,9 @@ def reindexFlags(
freq : {None, str},default None
The freq determines the projection range for the projection method. See above description for more details.
Defaultly (None), the sampling frequency of source is used.
- to_drop : {None, str, List[str]}, default None
+ to_mask : {None, str, List[str]}, default None
Flags referring to values that are to drop before flags projection. Relevant only when projecting with an
inverted shift method. Defaultly BAD is listed.
- freq_check : {None, 'check', 'auto'}, default None
- - None: do not validate frequency-string passed to `freq`
- - 'check': estimate frequency and log a warning if estimate miss matchs frequency string passed to 'freq', or
- if no uniform sampling rate could be estimated
- - 'auto': estimate frequency and use estimate. (Ignores `freq` parameter.)
Returns
-------
@@ -692,37 +706,20 @@ def reindexFlags(
Flags values and shape may have changed relatively to the flagger input.
"""
- # TODO: This needs a refactoring
- raise NotImplementedError("currently not available - rewrite needed")
+ if to_mask is None:
+ to_mask = BAD
- flagscol, metacols = flagger.getFlags(source, full=True)
+ flagscol = flagger[source]
if flagscol.empty:
return data, flagger
target_datcol = data[field]
- target_flagscol, target_metacols = flagger.getFlags(field, full=True)
-
- if (freq is None) and (method != "match"):
- freq_check = 'auto'
-
- freq = evalFreqStr(freq, freq_check, flagscol.index)
+ target_flagscol = flagger[field]
if method[-13:] == "interpolation":
- backprojected = flagscol.reindex(target_flagscol.index, method="bfill", tolerance=freq)
- fwrdprojected = flagscol.reindex(target_flagscol.index, method="ffill", tolerance=freq)
- b_replacement_mask = (backprojected > target_flagscol) & (backprojected >= fwrdprojected)
- f_replacement_mask = (fwrdprojected > target_flagscol) & (fwrdprojected > backprojected)
- target_flagscol.loc[b_replacement_mask] = backprojected.loc[b_replacement_mask]
- target_flagscol.loc[f_replacement_mask] = fwrdprojected.loc[f_replacement_mask]
-
- backprojected_meta = {}
- fwrdprojected_meta = {}
- for meta_key in target_metacols.keys():
- backprojected_meta[meta_key] = metacols[meta_key].reindex(target_metacols[meta_key].index, method='bfill',
- tolerance=freq)
- fwrdprojected_meta[meta_key] = metacols[meta_key].reindex(target_metacols[meta_key].index, method='ffill',
- tolerance=freq)
- target_metacols[meta_key].loc[b_replacement_mask] = backprojected_meta[meta_key].loc[b_replacement_mask]
- target_metacols[meta_key].loc[f_replacement_mask] = fwrdprojected_meta[meta_key].loc[f_replacement_mask]
+ ignore = _getChunkBounds(target_datcol, flagscol, freq)
+ flagscol[ignore] = np.nan
+ target_flagscol = _inverseInterpolation(target_flagscol, flagscol, freq)
+
if method[-3:] == "agg" or method == "match":
# Aggregation - Inversion
--
GitLab
From 2ef07abc28a1310c7749604545b6a18d5088abe8 Mon Sep 17 00:00:00 2001
From: Peter Luenenschloss <peter.luenenschloss@ufz.de>
Date: Tue, 16 Mar 2021 11:00:16 +0100
Subject: [PATCH 049/180] mergeHistoryByFunc added to the history manipulation
tools
---
saqc/flagger/flags.py | 36 ++++++++++++++++++++++++++++++++++++
saqc/funcs/resampling.py | 11 ++++++-----
2 files changed, 42 insertions(+), 5 deletions(-)
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index 537cf9178..7754995d7 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -380,5 +380,41 @@ def applyFunctionOnHistory(flags: Flags, column, hist_func, hist_kws, mask_func,
return flags
+def mergeHistoryByFunc(flags: Flags, field, source, merge_func, merge_func_kws, last_column=None):
+ """
+ Merges the information of one history (source) into the other (field). (Without altering fields indices)
+
+ Field indices remain unchanged. The merge is performed, via manipulating the field history values
+ column wise according to `merge_func`.
+
+
+ Parameters
+ ----------
+ flags
+ field
+ source
+ merge_func
+ merge_func_kws
+ last_column
+
+ Returns
+ -------
+
+ """
+ flags = flags.copy()
+ target_history = flags.history[field]
+ source_history = flags.history[source]
+ new_target_history = History()
+ for k in target_history.columns:
+ col_args_h = dict(source_col=source_history.hist[k])
+ col_args_m = dict(source_col=source_history.mask[k])
+ col_args_h.update(merge_func_kws)
+ col_args_m.update(merge_func_kws)
+ new_target_history.hist[k] = merge_func(target_history.hist[k], **col_args_h)
+ new_target_history.mask[k] = merge_func(target_history.mask[k][k], **col_args_m)
+
+ flags.history[field] = new_target_history
+ return flags
+
# for now we keep this name
Flagger = Flags
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index b7fb72135..5338ec34e 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -18,7 +18,7 @@ from saqc.funcs.tools import copy, drop, rename
from saqc.funcs.interpolation import interpolateIndex
from saqc.lib.tools import getDropMask, evalFreqStr
from saqc.lib.ts_operators import shift2Freq, aggregate2Freq
-from saqc.flagger.flags import applyFunctionOnHistory
+from saqc.flagger.flags import applyFunctionOnHistory, mergeHistoryByFunc
from saqc.lib.rolling import customRoller
logger = logging.getLogger("SaQC")
@@ -624,9 +624,9 @@ def _getChunkBounds(target_datcol, flagscol, freq):
return ignore_flags
-def _inverseInterpolation(target_flagscol, flagscol=None, freq=None):
- backprojected = flagscol.reindex(target_flagscol.index, method="bfill", tolerance=freq)
- fwrdprojected = flagscol.reindex(target_flagscol.index, method="ffill", tolerance=freq)
+def _inverseInterpolation(target_flagscol, source_col=None, freq=None):
+ backprojected = source_col.reindex(target_flagscol.index, method="bfill", tolerance=freq)
+ fwrdprojected = source_col.reindex(target_flagscol.index, method="ffill", tolerance=freq)
b_replacement_mask = (backprojected > target_flagscol) & (backprojected >= fwrdprojected)
f_replacement_mask = (fwrdprojected > target_flagscol) & (fwrdprojected > backprojected)
target_flagscol.loc[b_replacement_mask] = backprojected.loc[b_replacement_mask]
@@ -712,6 +712,7 @@ def reindexFlags(
flagscol = flagger[source]
if flagscol.empty:
return data, flagger
+
target_datcol = data[field]
target_flagscol = flagger[field]
@@ -719,7 +720,7 @@ def reindexFlags(
ignore = _getChunkBounds(target_datcol, flagscol, freq)
flagscol[ignore] = np.nan
target_flagscol = _inverseInterpolation(target_flagscol, flagscol, freq)
-
+ flagger = mergeHistoryByFunc(flagger, field, source, _inverseInterpolation, dict(freq=freq))
if method[-3:] == "agg" or method == "match":
# Aggregation - Inversion
--
GitLab
From c0d6bf54fc713b2da6627e1f990c8386153ee9b6 Mon Sep 17 00:00:00 2001
From: Peter Luenenschloss <peter.luenenschloss@ufz.de>
Date: Tue, 16 Mar 2021 13:24:00 +0100
Subject: [PATCH 050/180] flags backprojection translated to new flagger
concept
---
saqc/flagger/flags.py | 7 ++-
saqc/funcs/interpolation.py | 9 +--
saqc/funcs/resampling.py | 122 ++++++++++++++++--------------------
3 files changed, 66 insertions(+), 72 deletions(-)
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index 7754995d7..6f4caf4f9 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -411,7 +411,12 @@ def mergeHistoryByFunc(flags: Flags, field, source, merge_func, merge_func_kws,
col_args_h.update(merge_func_kws)
col_args_m.update(merge_func_kws)
new_target_history.hist[k] = merge_func(target_history.hist[k], **col_args_h)
- new_target_history.mask[k] = merge_func(target_history.mask[k][k], **col_args_m)
+ new_target_history.mask[k] = merge_func(target_history.mask[k], **col_args_m)
+
+ if last_column is None:
+ new_target_history.mask.iloc[:, -1:] = True
+ else:
+ new_target_history.append(last_column, force=True)
flags.history[field] = new_target_history
return flags
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index fecb5718d..95fb49970 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -249,18 +249,20 @@ def interpolateIndex(
to_mask = BAD
datcol = data[field]
+ flagscol = flagger[field]
if datcol.empty:
return data, flagger
start, end = datcol.index[0].floor(freq), datcol.index[-1].ceil(freq)
+ grid_index = pd.date_range(start=start, end=end, freq=freq, name=datcol.index.name)
+
datcol = datcol.copy()
+ datcol.drop(flagscol[flagscol >= to_mask].index, inplace=True)
datcol.dropna(inplace=True)
-
- grid_index = pd.date_range(start=start, end=end, freq=freq, name=datcol.index.name)
dat_index = datcol.index
# account for annoying case of subsequent frequency aligned values,
- # which differ exactly by the margin of 2*freq
+ # that differ exactly by the margin of 2*freq
gaps = ((dat_index[1:] - dat_index[:-1]) == 2*pd.Timedelta(freq))
gaps = dat_index[1:][gaps]
aligned_gaps = gaps.join(grid_index, how='inner')
@@ -286,7 +288,6 @@ def interpolateIndex(
data[field] = inter_data[grid_index]
# flags reshaping
- flagscol = flagger[field]
flagscol.drop(flagscol[flagscol >= to_mask].index, inplace=True)
flagscol = _overlap_rs(flagscol, freq, UNFLAGGED)
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 5338ec34e..49d0888b9 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -16,7 +16,7 @@ from saqc.core.register import register
from saqc.flagger import Flagger, initFlagsLike, History
from saqc.funcs.tools import copy, drop, rename
from saqc.funcs.interpolation import interpolateIndex
-from saqc.lib.tools import getDropMask, evalFreqStr
+from saqc.lib.tools import getDropMask, evalFreqStr, getFreqDelta
from saqc.lib.ts_operators import shift2Freq, aggregate2Freq
from saqc.flagger.flags import applyFunctionOnHistory, mergeHistoryByFunc
from saqc.lib.rolling import customRoller
@@ -624,7 +624,9 @@ def _getChunkBounds(target_datcol, flagscol, freq):
return ignore_flags
-def _inverseInterpolation(target_flagscol, source_col=None, freq=None):
+def _inverseInterpolation(target_flagscol, source_col=None, freq=None, chunk_bounds=None):
+ source_col = source_col.copy()
+ source_col[chunk_bounds] = np.nan
backprojected = source_col.reindex(target_flagscol.index, method="bfill", tolerance=freq)
fwrdprojected = source_col.reindex(target_flagscol.index, method="ffill", tolerance=freq)
b_replacement_mask = (backprojected > target_flagscol) & (backprojected >= fwrdprojected)
@@ -634,6 +636,39 @@ def _inverseInterpolation(target_flagscol, source_col=None, freq=None):
return target_flagscol
+def _inverseAggregation(target_flagscol, source_col=None, freq=None, method=None):
+ source_col = source_col.reindex(target_flagscol.index, method=method, tolerance=freq)
+ replacement_mask = source_col > target_flagscol
+ target_flagscol.loc[replacement_mask] = source_col.loc[replacement_mask]
+ return target_flagscol
+
+
+def _inverseShift(target_flagscol, source_col=None, freq=None, method=None, drop_mask=None):
+ target_flagscol_drops = target_flagscol[drop_mask]
+ target_flagscol.drop(drop_mask[drop_mask].index, inplace=True)
+ flags_merged = pd.merge_asof(
+ source_col,
+ pd.Series(target_flagscol.index.values, index=target_flagscol.index, name="pre_index"),
+ left_index=True,
+ right_index=True,
+ tolerance=freq,
+ direction=method,
+ )
+ flags_merged.dropna(subset=["pre_index"], inplace=True)
+ flags_merged = flags_merged.set_index(["pre_index"]).squeeze()
+
+ # write flags to target
+ replacement_mask = flags_merged > target_flagscol.loc[flags_merged.index]
+ target_flagscol.loc[replacement_mask[replacement_mask].index] = flags_merged.loc[replacement_mask]
+
+ # reinsert drops
+ target_flagscol = target_flagscol.reindex(target_flagscol.index.join(target_flagscol_drops.index, how="outer"))
+ target_flagscol.loc[target_flagscol_drops.index] = target_flagscol_drops.values
+
+ return target_flagscol
+
+
+
@register(masking='none', module="resampling")
def reindexFlags(
data: DictOfSeries,
@@ -713,81 +748,34 @@ def reindexFlags(
if flagscol.empty:
return data, flagger
+ if freq is None:
+ freq = getFreqDelta(flagscol.index)
+ if freq is None and not method=='match':
+ raise ValueError('To project irregularly sampled data, either use method="match", or pass custom '
+ 'projection range to freq parameter')
+
target_datcol = data[field]
target_flagscol = flagger[field]
-
+ append_dummy = pd.Series(np.nan, target_flagscol.index)
if method[-13:] == "interpolation":
ignore = _getChunkBounds(target_datcol, flagscol, freq)
- flagscol[ignore] = np.nan
- target_flagscol = _inverseInterpolation(target_flagscol, flagscol, freq)
- flagger = mergeHistoryByFunc(flagger, field, source, _inverseInterpolation, dict(freq=freq))
+ merge_func = _inverseInterpolation
+ merge_dict = dict(freq=freq, chunk_bounds=ignore)
+
if method[-3:] == "agg" or method == "match":
- # Aggregation - Inversion
projection_method = METHOD2ARGS[method][0]
tolerance = METHOD2ARGS[method][1](freq)
- flagscol = flagscol.reindex(target_flagscol.index, method=projection_method, tolerance=tolerance)
- replacement_mask = flagscol > target_flagscol
- target_flagscol.loc[replacement_mask] = flagscol.loc[replacement_mask]
- for meta_key in target_metacols.keys():
- metacols[meta_key] = metacols[meta_key].reindex(target_metacols[meta_key].index, method=projection_method,
- tolerance=tolerance)
- target_metacols[meta_key].loc[replacement_mask] = metacols[meta_key].loc[replacement_mask]
+ merge_func = _inverseAggregation
+ merge_dict = dict(freq=tolerance, method=projection_method)
+
if method[-5:] == "shift":
- # NOTE: although inverting a simple shift seems to be a less complex operation, it has quite some
- # code assigned to it and appears to be more verbose than inverting aggregation -
- # that owes itself to the problem of BAD/invalid values blocking a proper
- # shift inversion and having to be outsorted before shift inversion and re-inserted afterwards.
- #
- # starting with the dropping and its memorization:
-
- drop_mask = getDropMask(field, to_drop, flagger, BAD)
- drop_mask |= target_datcol.isna()
- target_flagscol_drops = target_flagscol[drop_mask]
- target_flagscol.drop(drop_mask[drop_mask].index, inplace=True)
-
- # shift inversion
+ drop_mask = (target_datcol.isna() | target_flagscol >= to_mask)
projection_method = METHOD2ARGS[method][0]
tolerance = METHOD2ARGS[method][1](freq)
- flags_merged = pd.merge_asof(
- flagscol,
- pd.Series(target_flagscol.index.values, index=target_flagscol.index, name="pre_index"),
- left_index=True,
- right_index=True,
- tolerance=tolerance,
- direction=projection_method,
- )
- flags_merged.dropna(subset=["pre_index"], inplace=True)
- flags_merged = flags_merged.set_index(["pre_index"]).squeeze()
-
- # write flags to target
- replacement_mask = flags_merged > target_flagscol.loc[flags_merged.index]
- target_flagscol.loc[replacement_mask[replacement_mask].index] = flags_merged.loc[replacement_mask]
-
- # reinsert drops
- target_flagscol = target_flagscol.reindex(target_flagscol.index.join(target_flagscol_drops.index, how="outer"))
- target_flagscol.loc[target_flagscol_drops.index] = target_flagscol_drops.values
-
- for meta_key in target_metacols.keys():
- target_metadrops = target_metacols[meta_key][drop_mask]
- target_metacols[meta_key].drop(drop_mask[drop_mask].index, inplace=True)
- meta_merged = pd.merge_asof(
- metacols[meta_key],
- pd.Series(target_metacols[meta_key].index.values, index=target_metacols[meta_key].index,
- name="pre_index"),
- left_index=True,
- right_index=True,
- tolerance=tolerance,
- direction=projection_method,
- )
- meta_merged.dropna(subset=["pre_index"], inplace=True)
- meta_merged = meta_merged.set_index(["pre_index"]).squeeze()
- # reinsert drops
- target_metacols[meta_key][replacement_mask[replacement_mask].index] = meta_merged[replacement_mask]
- target_metacols[meta_key] = target_metacols[meta_key].reindex(
- target_metacols[meta_key].index.join(target_metadrops.index, how="outer"))
- target_metacols[meta_key].loc[target_metadrops.index] = target_metadrops.values
-
- flagger = flagger.setFlags(field, flag=target_flagscol, with_extra=True, **target_metacols, **kwargs)
+ merge_func = _inverseShift
+ merge_dict = dict(freq=tolerance, method=projection_method, drop_mask=drop_mask)
+
+ flagger = mergeHistoryByFunc(flagger, field, source, merge_func, merge_dict, last_column=append_dummy)
return data, flagger
--
GitLab
From 62e6b76434dca6e80758a8c46fb65aa09e87e9e4 Mon Sep 17 00:00:00 2001
From: Peter Luenenschloss <peter.luenenschloss@ufz.de>
Date: Tue, 16 Mar 2021 14:10:33 +0100
Subject: [PATCH 051/180] ...
---
saqc/flagger/flags.py | 4 +++-
test/funcs/test_harm_funcs.py | 4 +++-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index 6f4caf4f9..62d8d3292 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -405,7 +405,9 @@ def mergeHistoryByFunc(flags: Flags, field, source, merge_func, merge_func_kws,
target_history = flags.history[field]
source_history = flags.history[source]
new_target_history = History()
- for k in target_history.columns:
+ import pdb
+ pdb.set_trace()
+ for k in target_history.hist.columns:
col_args_h = dict(source_col=source_history.hist[k])
col_args_m = dict(source_col=source_history.mask[k])
col_args_h.update(merge_func_kws)
diff --git a/test/funcs/test_harm_funcs.py b/test/funcs/test_harm_funcs.py
index 41408611f..d0e72553f 100644
--- a/test/funcs/test_harm_funcs.py
+++ b/test/funcs/test_harm_funcs.py
@@ -54,7 +54,9 @@ def test_harmSingleVarIntermediateFlagging(data, flagger, reshaper):
field = data.columns[0]
data, flagger = linear(data, "data", flagger, freq)
# flag something bad
- flagger = flagger.setFlags("data", loc=data[field].index[3:4])
+ f_ser = pd.Series(data=[-np.inf] * len(data[field]), index=data[field].index)
+ f_ser[3:4] = BAD
+ flagger[field] = f_ser
data, flagger = mapToOriginal(data, "data", flagger, method="inverse_" + reshaper)
d = data[field]
if reshaper == "nagg":
--
GitLab
From e124ceaa028cce2c45c41dbfc89675288242aafc Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 16 Mar 2021 17:42:05 +0100
Subject: [PATCH 052/180] reworked register.py, fixed `to_mask` defaults
---
saqc/core/core.py | 3 +-
saqc/core/register.py | 137 ++++++++++++++++++++++++++++--------------
2 files changed, 93 insertions(+), 47 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 3fdd00a5a..058ed9c18 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -228,7 +228,8 @@ class SaQC(FuncModules):
def inner(field: str, *fargs, target: str = None, regex: bool = False, plot: bool = False, inplace: bool = False, **fkwargs) -> SaQC:
- fkwargs.setdefault('to_mask', self._to_mask)
+ if self._to_mask is not None:
+ fkwargs.setdefault('to_mask', self._to_mask)
control = APIController(
plot=plot
diff --git a/saqc/core/register.py b/saqc/core/register.py
index fcdcf5049..0d423fba5 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -7,6 +7,7 @@ import dataclasses
import numpy as np
import pandas as pd
import dios
+import warnings
from saqc.common import *
from saqc.core.lib import SaQCFunction
@@ -33,7 +34,7 @@ class CallCtrl:
kwargs: dict
masking: MaskingStrT = None
- to_mask: float = None
+ mthresh: float = None
mask: dios.DictOfSeries = None
@@ -41,18 +42,18 @@ def register(masking: MaskingStrT = "all", module: Optional[str] = None):
# executed on module import
def inner(func):
+ func_name = func.__name__
+ if module:
+ func_name = f"{module}.{func_name}"
# executed if a register-decorated function is called,
# nevertheless if it is called plain or via `SaQC.func`.
@wraps(func)
def callWrapper(*args, **kwargs):
- args, kwargs, ctrl = _preCall(func, args, kwargs, masking)
+ args, kwargs, ctrl = _preCall(func, args, kwargs, masking, func_name)
result = func(*args, **kwargs)
- return _postCall(result, ctrl)
+ return _postCall(result, ctrl, func_name)
- func_name = func.__name__
- if module:
- func_name = f"{module}.{func_name}"
FUNC_MAP[func_name] = SaQCFunction(func_name, callWrapper)
return callWrapper
@@ -60,7 +61,7 @@ def register(masking: MaskingStrT = "all", module: Optional[str] = None):
return inner
-def _preCall(func: callable, args: tuple, kwargs: dict, masking: MaskingStrT):
+def _preCall(func: callable, args: tuple, kwargs: dict, masking: MaskingStrT, fname: str):
"""
Handler that runs before any call to a saqc-function.
@@ -95,15 +96,18 @@ def _preCall(func: callable, args: tuple, kwargs: dict, masking: MaskingStrT):
control keyword-arguments passed to `_postCall`
"""
- kwargs.setdefault('to_mask', None)
- data, field, flagger, *args = args
+ mthresh = _getMaskingThresh(masking, kwargs, fname)
+ kwargs['to_mask'] = mthresh
- ctrl = CallCtrl(func, data.copy(), field, flagger.copy(), args, kwargs, masking=masking)
+ data, field, flagger, *args = args
+ ctrl = CallCtrl(func, data.copy(), field, flagger.copy(), args, kwargs, masking=masking, mthresh=mthresh)
# masking
- ctrl.to_mask = _getToMask(ctrl)
columns = _getMaskingColumns(ctrl, ctrl.masking)
- data, ctrl.mask = _maskData(data, flagger, columns, ctrl.to_mask)
+ data, mask = _maskData(data, flagger, columns, mthresh)
+
+ # store mask
+ ctrl.mask = mask
# flags
flagger = _prepareFlags(flagger, ctrl)
@@ -112,7 +116,7 @@ def _preCall(func: callable, args: tuple, kwargs: dict, masking: MaskingStrT):
return args, kwargs, ctrl
-def _postCall(result, ctrl: CallCtrl) -> FuncReturnT:
+def _postCall(result, ctrl: CallCtrl, fname: str) -> FuncReturnT:
"""
Handler that runs after any call to a saqc-function.
@@ -123,9 +127,13 @@ def _postCall(result, ctrl: CallCtrl) -> FuncReturnT:
----------
result : tuple
the result from the called function, namely: data and flagger
+
ctrl : dict
control keywords from `_preCall`
+ fname : str
+ Name of the (just) called saqc-function
+
Returns
-------
data, flagger : dios.DictOfSeries, saqc.flagger.Flagger
@@ -152,35 +160,68 @@ def _getMaskingColumns(ctrl: CallCtrl, masking: MaskingStrT):
raise ValueError(f"wrong use of `register(masking={ctrl.masking})`")
-def _getToMask(ctrl):
- to_mask = ctrl.kwargs['to_mask']
- _warnForUnusedMasking(ctrl.masking, to_mask)
+def _getMaskingThresh(masking, kwargs, fname):
+ """
+ Check the correct usage of the `to_mask` keyword, iff passed, otherwise return a default.
- if to_mask is None:
- to_mask = UNFLAGGED
+ Parameters
+ ----------
+ masking : str
+ The function-scope masking keyword a saqc-function is decorated with.
+ kwargs : dict
+ The kwargs that will be passed to the saqc-function, possibly contain ``to_mask``.
+ fname : str
+ The name of the saqc-function to be called later (not here), to use in meaningful
+ error messages
- return to_mask
+ Returns
+ -------
+ threshold: float
+ All data gets masked, if the flags are equal or worse than the threshold.
+
+ Notes
+ -----
+ If ``to_mask`` is **not** in the kwargs, the threshold defaults to
+ - ``-np.inf``
+ If boolean ``to_mask`` is found in the kwargs, the threshold defaults to
+ - ``-np.inf``, if ``True``
+ - ``+np.inf``, if ``False``
+ If a floatish ``to_mask`` is found in the kwargs, this value is taken as the threshold.
+ """
+ if 'to_mask' not in kwargs:
+ return UNFLAGGED
+
+ thresh = kwargs['to_mask']
+
+ if not isinstance(thresh, (bool, float, int)):
+ raise TypeError(f"'to_mask' must be of type bool or float")
+ if masking == 'none' and thresh not in (False, np.inf):
+ # TODO: fix warning reference to docu
+ warnings.warn(f"the saqc-function {fname!r} ignore masking and therefore does not evaluate the passed "
+ f"'to_mask'-keyword. Please refer to the documentation: TODO")
-def _warnForUnusedMasking(masking, to_mask):
- # warn if the user explicitly pass `to_mask=..` to a function that is
- # decorated by `register(masking='none')`, by which `to_mask` is ignored
- # TODO: fix warning message
- if masking == 'none' and to_mask not in (None, np.inf):
- logging.warning("`to_mask` is given, but the saqc-function ignore masking."
- " Please refer to the documentation: TODO")
+ if thresh is True: # masking ON
+ thresh = UNFLAGGED
+
+ if thresh is False: # masking OFF
+ thresh = np.inf
+
+ thresh = float(thresh) # handle int
+
+ return thresh
# TODO: this is heavily undertested
-def _maskData(data, flagger, columns, to_mask) -> Tuple[dios.DictOfSeries, dios.DictOfSeries]:
+def _maskData(data, flagger, columns, thresh) -> Tuple[dios.DictOfSeries, dios.DictOfSeries]:
"""
- Mask data with Nans by flags, according to masking and to_mask.
+ Mask data with Nans by flags worse that a threshold and according to masking keyword in decorator.
"""
mask = dios.DictOfSeries(columns=columns)
# we use numpy here because it is faster
for c in columns:
- col_mask = _getMask(flagger[c].to_numpy(), to_mask)
+ col_mask = _getMask(flagger[c].to_numpy(), thresh)
if any(col_mask):
col_data = data[c].to_numpy(dtype=np.float64)
@@ -192,12 +233,14 @@ def _maskData(data, flagger, columns, to_mask) -> Tuple[dios.DictOfSeries, dios.
return data, mask
-def _getMask(flags: Union[np.array, pd.Series], to_mask: float) -> Union[np.array, pd.Series]:
+def _getMask(flags: Union[np.array, pd.Series], thresh: float) -> Union[np.array, pd.Series]:
"""
- Return a mask of flags accordingly to `to_mask`.
- Return type is same as flags.
+ Return a mask of flags accordingly to `thresh`. Return type is same as flags.
"""
- return flags > to_mask
+ if thresh == UNFLAGGED:
+ return flags > UNFLAGGED
+
+ return flags >= thresh
def _prepareFlags(flagger: Flagger, ctrl: CallCtrl) -> Flagger:
@@ -213,18 +256,20 @@ def _prepareFlags(flagger: Flagger, ctrl: CallCtrl) -> Flagger:
def _restoreFlags(flagger: Flagger, ctrl: CallCtrl):
if ctrl.masking == 'none':
- ctrl.flagger = flagger
+ return flagger
+
+ result = ctrl.flagger
- else:
- columns = flagger.columns
- if ctrl.masking == 'field':
- columns = columns.difference(ctrl.flagger.columns)
- columns = columns.append(pd.Index([ctrl.field]))
+ columns = flagger.columns
+ # take field column and all possibly newly added columns
+ if ctrl.masking == 'field':
+ columns = columns.difference(ctrl.flagger.columns)
+ columns = columns.append(pd.Index([ctrl.field]))
- for c in columns:
- ctrl.flagger[c] = flagger[c]
+ for c in columns:
+ result[c] = flagger[c]
- return ctrl.flagger
+ return result
# TODO: this is heavily undertested
@@ -273,15 +318,15 @@ def _unmaskData(data: dios.DictOfSeries, ctrl: CallCtrl) -> dios.DictOfSeries:
if not old.data[c].index.equals(data[c].index):
continue
- restore_old_val = old.mask[c].to_numpy() & data[c].isna().to_numpy()
+ restore_old_mask = old.mask[c].to_numpy() & data[c].isna().to_numpy()
# we have nothing to restore
- if not any(restore_old_val):
+ if not any(restore_old_mask):
continue
# restore old values if no new are present
- ol, nw = old.data[c].to_numpy(), data[c].to_numpy()
- data.loc[:, c] = np.where(restore_old_val, ol, nw)
+ v_old, v_new = old.data[c].to_numpy(), data[c].to_numpy()
+ data.loc[:, c] = np.where(restore_old_mask, v_old, v_new)
return data
--
GitLab
From 2abc83bf6ddf86a9bded6079b2b9ba3e35aaed55 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 16 Mar 2021 18:08:58 +0100
Subject: [PATCH 053/180] simplyfied register, according to @schaefed 's
suggestions
---
saqc/core/register.py | 28 ++++++++++++++++++----------
1 file changed, 18 insertions(+), 10 deletions(-)
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 0d423fba5..23ec179a4 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -102,15 +102,15 @@ def _preCall(func: callable, args: tuple, kwargs: dict, masking: MaskingStrT, fn
data, field, flagger, *args = args
ctrl = CallCtrl(func, data.copy(), field, flagger.copy(), args, kwargs, masking=masking, mthresh=mthresh)
- # masking
- columns = _getMaskingColumns(ctrl, ctrl.masking)
+ # handle data - masking
+ columns = _getMaskingColumns(data, field, masking)
data, mask = _maskData(data, flagger, columns, mthresh)
# store mask
ctrl.mask = mask
- # flags
- flagger = _prepareFlags(flagger, ctrl)
+ # handle flags - clearing
+ flagger = _prepareFlags(flagger, masking)
args = data, field, flagger, *args
return args, kwargs, ctrl
@@ -144,18 +144,23 @@ def _postCall(result, ctrl: CallCtrl, fname: str) -> FuncReturnT:
return data, flagger
-def _getMaskingColumns(ctrl: CallCtrl, masking: MaskingStrT):
+def _getMaskingColumns(data: dios.DictOfSeries, field: str, masking: MaskingStrT):
"""
+ Returns
+ -------
+ columns: pd.Index
+ Data columns that need to be masked.
+
Raises
------
ValueError: if given masking literal is not supported
"""
if masking == 'all':
- return ctrl.data.columns
+ return data.columns
if masking == 'none':
return pd.Index([])
if masking == 'field':
- return pd.Index([ctrl.field])
+ return pd.Index([field])
raise ValueError(f"wrong use of `register(masking={ctrl.masking})`")
@@ -243,12 +248,12 @@ def _getMask(flags: Union[np.array, pd.Series], thresh: float) -> Union[np.array
return flags >= thresh
-def _prepareFlags(flagger: Flagger, ctrl: CallCtrl) -> Flagger:
+def _prepareFlags(flagger: Flagger, masking) -> Flagger:
"""
Clear flags before each call.
"""
- # either the index or the columns itself changed
- if ctrl.masking == 'none':
+ # Either the index or the columns itself changed
+ if masking == 'none':
return flagger
return initFlagsLike(flagger, initial_value=UNTOUCHED)
@@ -267,6 +272,9 @@ def _restoreFlags(flagger: Flagger, ctrl: CallCtrl):
columns = columns.append(pd.Index([ctrl.field]))
for c in columns:
+ # this implicitly squash the new-flagger history (RHS) to a single column, which than is appended to
+ # the old history (LHS). The new-flagger history possibly consist of multiple columns, one for each
+ # time flags was set to the flagger.
result[c] = flagger[c]
return result
--
GitLab
From dbad9cc451bc949a32163c442d86307f726d3667 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 16 Mar 2021 18:12:14 +0100
Subject: [PATCH 054/180] renamed common.py to constants.py
---
saqc/__init__.py | 2 +-
saqc/{common.py => constants.py} | 0
saqc/core/core.py | 2 +-
saqc/core/modules/flagtools.py | 2 +-
saqc/core/modules/interpolation.py | 2 +-
saqc/core/register.py | 2 +-
saqc/core/visitor.py | 2 +-
saqc/flagger/flags.py | 2 +-
saqc/flagger/history.py | 2 +-
saqc/funcs/changepoints.py | 2 +-
saqc/funcs/flagtools.py | 2 +-
saqc/funcs/generic.py | 2 +-
saqc/funcs/interpolation.py | 2 +-
saqc/funcs/outliers.py | 2 +-
saqc/funcs/resampling.py | 2 +-
saqc/funcs/scores.py | 2 +-
saqc/funcs/tools.py | 2 +-
saqc/lib/plotting.py | 2 +-
tests/common.py | 2 +-
tests/core/test_core.py | 2 +-
tests/flagger/test_flags.py | 2 +-
tests/funcs/test_constants_detection.py | 2 +-
tests/funcs/test_functions.py | 2 +-
tests/funcs/test_generic_api_functions.py | 2 +-
tests/funcs/test_generic_config_functions.py | 2 +-
tests/funcs/test_pattern_rec.py | 2 +-
tests/funcs/test_proc_functions.py | 2 +-
tests/funcs/test_spikes_detection.py | 2 +-
tests/fuzzy/init.py | 2 +-
tests/fuzzy/test_masking.py | 2 +-
30 files changed, 29 insertions(+), 29 deletions(-)
rename saqc/{common.py => constants.py} (100%)
diff --git a/saqc/__init__.py b/saqc/__init__.py
index 0c2bb6d4b..d7155f47a 100644
--- a/saqc/__init__.py
+++ b/saqc/__init__.py
@@ -4,7 +4,7 @@
__version__ = "1.4"
# import order: from small to big
-from saqc.common import *
+from saqc.constants import *
from saqc.flagger import *
from saqc.core.register import register
from saqc.core.core import SaQC
diff --git a/saqc/common.py b/saqc/constants.py
similarity index 100%
rename from saqc/common.py
rename to saqc/constants.py
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 058ed9c18..3d4a7517c 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -17,7 +17,7 @@ import numpy as np
import timeit
import inspect
-from saqc.common import *
+from saqc.constants import *
from saqc.flagger import initFlagsLike, Flagger
from saqc.core.lib import APIController, ColumnSelector
from saqc.core.register import FUNC_MAP, SaQCFunction
diff --git a/saqc/core/modules/flagtools.py b/saqc/core/modules/flagtools.py
index c9c1b0892..637dc6f85 100644
--- a/saqc/core/modules/flagtools.py
+++ b/saqc/core/modules/flagtools.py
@@ -8,7 +8,7 @@ import pandas as pd
from dios.dios import DictOfSeries
from saqc.core.modules.base import ModuleBase
-from saqc.common import *
+from saqc.constants import *
class FlagTools(ModuleBase):
diff --git a/saqc/core/modules/interpolation.py b/saqc/core/modules/interpolation.py
index c73da2563..42db500c2 100644
--- a/saqc/core/modules/interpolation.py
+++ b/saqc/core/modules/interpolation.py
@@ -7,7 +7,7 @@ from typing_extensions import Literal
import numpy as np
import pandas as pd
-from saqc.common import *
+from saqc.constants import *
from saqc.core.modules.base import ModuleBase
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 23ec179a4..817f3b559 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -9,7 +9,7 @@ import pandas as pd
import dios
import warnings
-from saqc.common import *
+from saqc.constants import *
from saqc.core.lib import SaQCFunction
from saqc.lib.types import FuncReturnT
from saqc.flagger.flags import Flagger, initFlagsLike
diff --git a/saqc/core/visitor.py b/saqc/core/visitor.py
index 560d87748..c517261e0 100644
--- a/saqc/core/visitor.py
+++ b/saqc/core/visitor.py
@@ -6,7 +6,7 @@ import ast
import numpy as np
import pandas as pd
-from saqc.common import *
+from saqc.constants import *
from saqc.core.register import FUNC_MAP
import saqc.lib.ts_operators as ts_ops
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index 537cf9178..f5e36cc44 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -3,7 +3,7 @@
from __future__ import annotations
import dios
-from saqc.common import *
+from saqc.constants import *
from saqc.flagger.history import History
import pandas as pd
from typing import Union, Dict, DefaultDict, Optional, Type, Tuple, Iterable
diff --git a/saqc/flagger/history.py b/saqc/flagger/history.py
index 551dba6c4..72a573bd1 100644
--- a/saqc/flagger/history.py
+++ b/saqc/flagger/history.py
@@ -4,7 +4,7 @@ from __future__ import annotations
from typing import Tuple, Type
import pandas as pd
import numpy as np
-from saqc.common import *
+from saqc.constants import *
class History:
diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py
index 00d201f77..200711da4 100644
--- a/saqc/funcs/changepoints.py
+++ b/saqc/funcs/changepoints.py
@@ -11,7 +11,7 @@ from typing_extensions import Literal
from dios import DictOfSeries
-from saqc.common import *
+from saqc.constants import *
from saqc.core.register import register
from saqc.lib.tools import customRoller
from saqc.flagger import Flagger
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index c9227a7d4..5c2b341a9 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -7,7 +7,7 @@ import pandas as pd
from dios import DictOfSeries
-from saqc.common import *
+from saqc.constants import *
from saqc.lib.types import *
from saqc.core.register import register
from saqc.flagger import Flagger
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index 862b3b981..4678bec62 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -10,7 +10,7 @@ import pandas as pd
from dios import DictOfSeries
-from saqc.common import *
+from saqc.constants import *
from saqc.core.register import register
from saqc.core.visitor import ENVIRONMENT
from saqc.flagger import Flagger, initFlagsLike
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index a615aee92..5f59e2189 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -9,7 +9,7 @@ import pandas as pd
from dios import DictOfSeries
-from saqc.common import *
+from saqc.constants import *
from saqc.core.register import register
from saqc.flagger import Flagger
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index 818a80ed0..ab486487d 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -14,7 +14,7 @@ from outliers import smirnov_grubbs
from dios import DictOfSeries
-from saqc.common import *
+from saqc.constants import *
from saqc.core.register import register
from saqc.flagger import Flagger
from saqc.lib.tools import (
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 3a4d8ce78..d8469586e 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -11,7 +11,7 @@ import pandas as pd
from dios import DictOfSeries
-from saqc.common import *
+from saqc.constants import *
from saqc.core.register import register
from saqc.flagger import Flagger, initFlagsLike, History
from saqc.funcs.tools import copy, drop, rename
diff --git a/saqc/funcs/scores.py b/saqc/funcs/scores.py
index bdc27b597..1a49f4e1e 100644
--- a/saqc/funcs/scores.py
+++ b/saqc/funcs/scores.py
@@ -8,7 +8,7 @@ import pandas as pd
from dios import DictOfSeries
-from saqc.common import *
+from saqc.constants import *
from saqc.core.register import register
from saqc.flagger import Flagger
from saqc.lib import ts_operators as ts_ops
diff --git a/saqc/funcs/tools.py b/saqc/funcs/tools.py
index 3e49f762b..edbb4c80c 100644
--- a/saqc/funcs/tools.py
+++ b/saqc/funcs/tools.py
@@ -8,7 +8,7 @@ import numpy as np
from dios import DictOfSeries
-from saqc.common import *
+from saqc.constants import *
from saqc.core.register import register
from saqc.flagger import Flagger
from saqc.lib.tools import periodicMask
diff --git a/saqc/lib/plotting.py b/saqc/lib/plotting.py
index c03670b5c..5f79f28bb 100644
--- a/saqc/lib/plotting.py
+++ b/saqc/lib/plotting.py
@@ -8,7 +8,7 @@ import pandas as pd
import dios
import matplotlib.pyplot as plt
from typing import List, Dict, Optional
-from saqc.common import *
+from saqc.constants import *
from saqc.flagger import Flagger
diff --git a/tests/common.py b/tests/common.py
index f61ddac3e..3e70ff349 100644
--- a/tests/common.py
+++ b/tests/common.py
@@ -6,7 +6,7 @@ import numpy as np
import pandas as pd
import dios
-from saqc.common import *
+from saqc.constants import *
from saqc.flagger import Flagger, initFlagsLike
diff --git a/tests/core/test_core.py b/tests/core/test_core.py
index cddc2fd59..b9773a473 100644
--- a/tests/core/test_core.py
+++ b/tests/core/test_core.py
@@ -6,7 +6,7 @@ import pytest
import numpy as np
import pandas as pd
-from saqc.common import *
+from saqc.constants import *
from saqc.flagger import initFlagsLike
from saqc.funcs import flagRange
from saqc.lib import plotting as splot
diff --git a/tests/flagger/test_flags.py b/tests/flagger/test_flags.py
index 652022048..db40b7e6c 100644
--- a/tests/flagger/test_flags.py
+++ b/tests/flagger/test_flags.py
@@ -4,7 +4,7 @@ import pytest
import numpy as np
import pandas as pd
-from saqc.common import *
+from saqc.constants import *
from saqc.flagger.flags import Flags
from tests.flagger.test_history import (
diff --git a/tests/funcs/test_constants_detection.py b/tests/funcs/test_constants_detection.py
index 1ae7be198..da7b83836 100644
--- a/tests/funcs/test_constants_detection.py
+++ b/tests/funcs/test_constants_detection.py
@@ -4,7 +4,7 @@
import pytest
import numpy as np
-from saqc.common import *
+from saqc.constants import *
from saqc.funcs.constants import flagConstants, flagByVariance
from saqc.flagger import initFlagsLike
diff --git a/tests/funcs/test_functions.py b/tests/funcs/test_functions.py
index bbaa20ad5..47c8ae9d2 100644
--- a/tests/funcs/test_functions.py
+++ b/tests/funcs/test_functions.py
@@ -3,7 +3,7 @@
import dios
-from saqc.common import *
+from saqc.constants import *
from saqc.flagger import initFlagsLike
from saqc.funcs.drift import flagDriftFromNorm, flagDriftFromReference, flagDriftFromScaledNorm
from saqc.funcs.outliers import flagCrossStatistic, flagRange
diff --git a/tests/funcs/test_generic_api_functions.py b/tests/funcs/test_generic_api_functions.py
index d581c4344..8c3ce15ff 100644
--- a/tests/funcs/test_generic_api_functions.py
+++ b/tests/funcs/test_generic_api_functions.py
@@ -4,7 +4,7 @@
import pytest
import pandas as pd
-from saqc.common import *
+from saqc.constants import *
from saqc.core.register import register
from saqc.funcs.tools import mask
from saqc import SaQC
diff --git a/tests/funcs/test_generic_config_functions.py b/tests/funcs/test_generic_config_functions.py
index 6e84e6a99..fe8242cd2 100644
--- a/tests/funcs/test_generic_config_functions.py
+++ b/tests/funcs/test_generic_config_functions.py
@@ -7,7 +7,7 @@ import numpy as np
import pandas as pd
import dios
-from saqc.common import *
+from saqc.constants import *
from saqc.flagger import Flagger, initFlagsLike
from saqc.core.visitor import ConfigFunctionParser
from saqc.core.config import Fields as F
diff --git a/tests/funcs/test_pattern_rec.py b/tests/funcs/test_pattern_rec.py
index 3ca69d707..b434e3c24 100644
--- a/tests/funcs/test_pattern_rec.py
+++ b/tests/funcs/test_pattern_rec.py
@@ -5,7 +5,7 @@ import pytest
import pandas as pd
import dios
-from saqc.common import *
+from saqc.constants import *
from saqc.flagger import initFlagsLike
from saqc.funcs.pattern import *
from tests.common import initData
diff --git a/tests/funcs/test_proc_functions.py b/tests/funcs/test_proc_functions.py
index 31337badc..d7cada078 100644
--- a/tests/funcs/test_proc_functions.py
+++ b/tests/funcs/test_proc_functions.py
@@ -6,7 +6,7 @@
import dios
-from saqc.common import *
+from saqc.constants import *
from saqc.funcs.transformation import transform
from saqc.funcs.drift import correctOffset
from saqc.funcs.interpolation import interpolateByRolling, interpolateInvalid, interpolateIndex
diff --git a/tests/funcs/test_spikes_detection.py b/tests/funcs/test_spikes_detection.py
index 578dd9c44..faa256068 100644
--- a/tests/funcs/test_spikes_detection.py
+++ b/tests/funcs/test_spikes_detection.py
@@ -12,7 +12,7 @@ from saqc.funcs.outliers import (
flagMVScores,
flagByGrubbs,
)
-from saqc.common import *
+from saqc.constants import *
from saqc.flagger import initFlagsLike
diff --git a/tests/fuzzy/init.py b/tests/fuzzy/init.py
index adbbffdc5..ad93f02c6 100644
--- a/tests/fuzzy/init.py
+++ b/tests/fuzzy/init.py
@@ -21,7 +21,7 @@ from hypothesis.strategies import (
from hypothesis.extra.numpy import arrays, from_dtype
from hypothesis.strategies._internal.types import _global_type_lookup
-from saqc.common import *
+from saqc.constants import *
from saqc.core.register import FUNC_MAP
from saqc.core.lib import SaQCFunction
from saqc.lib.types import FreqString, ColumnName, IntegerWindow
diff --git a/tests/fuzzy/test_masking.py b/tests/fuzzy/test_masking.py
index 9d45520eb..cc7637099 100644
--- a/tests/fuzzy/test_masking.py
+++ b/tests/fuzzy/test_masking.py
@@ -7,7 +7,7 @@ import pandas as pd
from hypothesis import given, settings
-from saqc.common import *
+from saqc.constants import *
from saqc.flagger import Flagger
from saqc.core.register import _maskData, _unmaskData
--
GitLab
From 01aa3f282cbb379dbcbd4773efd1c835533b7642 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 17 Mar 2021 17:08:15 +0100
Subject: [PATCH 055/180] fixed to_mask usage
---
saqc/funcs/interpolation.py | 28 +++++++++++++---------------
saqc/funcs/resampling.py | 22 ++++------------------
2 files changed, 17 insertions(+), 33 deletions(-)
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index da8a10d4d..dd5036d9c 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -101,7 +101,6 @@ def interpolateInvalid(
flag: float = UNFLAGGED,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
-
"""
Function to interpolate nan values in the data.
@@ -185,15 +184,15 @@ def interpolateIndex(
field: str,
flagger: Flagger,
freq: str,
- method: Literal["linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric",
- "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"],
- inter_order: int=2,
- downgrade_interpolation: bool=False,
- inter_limit: int=2,
- to_mask: Optional[Union[Any, Sequence[Any]]]=BAD,
+ method: Literal[
+ "linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric",
+ "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"
+ ],
+ inter_order: int = 2,
+ downgrade_interpolation: bool = False,
+ inter_limit: int = 2,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
-
"""
Function to interpolate the data at regular (equidistant) timestamps (or Grid points).
@@ -244,19 +243,18 @@ def interpolateIndex(
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
+ if data[field].empty:
+ return data, flagger
- if to_mask is None:
- to_mask = BAD
-
- datcol = data[field]
+ datcol = data[field].copy()
flagscol = flagger[field]
- if datcol.empty:
- return data, flagger
start, end = datcol.index[0].floor(freq), datcol.index[-1].ceil(freq)
grid_index = pd.date_range(start=start, end=end, freq=freq, name=datcol.index.name)
- datcol = datcol.copy()
+ # always injected by register
+ to_mask = kwargs['to_mask']
+
datcol.drop(flagscol[flagscol >= to_mask].index, inplace=True)
datcol.dropna(inplace=True)
dat_index = datcol.index
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 9c45aa576..3e24cd505 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -23,7 +23,6 @@ from saqc.lib.rolling import customRoller
logger = logging.getLogger("SaQC")
-
METHOD2ARGS = {
"inverse_fshift": ("backward", pd.Timedelta),
"inverse_bshift": ("forward", pd.Timedelta),
@@ -565,9 +564,7 @@ def resample(
# create a dummys
if all_na_2_empty and datcol.dropna().empty:
- # Todo: This needs discussion. It makes possible, that different harmonized variables,
- # resulting from the harmonization of the same logger, have differing timestamps!
- # (Same holds for starting/ending nan-chunk truncation)
+ # Todo: This needs discussion. See issue #GL170
datcol = pd.Series([], index=pd.DatetimeIndex([]), name=field)
flagscol = pd.Series([], index=pd.DatetimeIndex([]), name=field)
@@ -668,7 +665,6 @@ def _inverseShift(target_flagscol, source_col=None, freq=None, method=None, drop
return target_flagscol
-
@register(masking='none', module="resampling")
def reindexFlags(
data: DictOfSeries,
@@ -676,11 +672,9 @@ def reindexFlags(
flagger: Flagger,
method: Literal["inverse_fagg", "inverse_bagg", "inverse_nagg", "inverse_fshift", "inverse_bshift", "inverse_nshift"],
source: str,
- freq: Optional[str]=None,
- to_mask: Optional[Union[Any, Sequence[Any]]]=BAD,
+ freq: Optional[str] = None,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
-
"""
The Function projects flags of "source" onto flags of "field". Wherever the "field" flags are "better" then the
source flags projected on them, they get overridden with this associated source flag value.
@@ -728,9 +722,6 @@ def reindexFlags(
freq : {None, str},default None
The freq determines the projection range for the projection method. See above description for more details.
Defaultly (None), the sampling frequency of source is used.
- to_mask : {None, str, List[str]}, default None
- Flags referring to values that are to drop before flags projection. Relevant only when projecting with an
- inverted shift method. Defaultly BAD is listed.
Returns
-------
@@ -740,17 +731,13 @@ def reindexFlags(
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
-
- if to_mask is None:
- to_mask = BAD
-
flagscol = flagger[source]
if flagscol.empty:
return data, flagger
if freq is None:
freq = getFreqDelta(flagscol.index)
- if freq is None and not method=='match':
+ if freq is None and not method == 'match':
raise ValueError('To project irregularly sampled data, either use method="match", or pass custom '
'projection range to freq parameter')
@@ -762,15 +749,14 @@ def reindexFlags(
merge_func = _inverseInterpolation
merge_dict = dict(freq=freq, chunk_bounds=ignore)
-
if method[-3:] == "agg" or method == "match":
projection_method = METHOD2ARGS[method][0]
tolerance = METHOD2ARGS[method][1](freq)
merge_func = _inverseAggregation
merge_dict = dict(freq=tolerance, method=projection_method)
-
if method[-5:] == "shift":
+ to_mask = kwargs['to_mask']
drop_mask = (target_datcol.isna() | target_flagscol >= to_mask)
projection_method = METHOD2ARGS[method][0]
tolerance = METHOD2ARGS[method][1](freq)
--
GitLab
From ab63adf5cd24c478c58174e9b786cde32c653775 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 19 Mar 2021 20:28:53 +0100
Subject: [PATCH 056/180] refactored register.py as @schaefed suggested,
renamed former CallCtrl to CallState and made it more KISSier
---
saqc/core/register.py | 101 ++++++++++++++++++++++++------------------
1 file changed, 57 insertions(+), 44 deletions(-)
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 817f3b559..9bb841b49 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -23,19 +23,19 @@ MaskingStrT = Literal["all", "field", "none"]
@dataclasses.dataclass
-class CallCtrl:
+class CallState:
func: callable
data: dios.DictOfSeries
- field: str
flagger: Flagger
+ field: str
args: tuple
kwargs: dict
- masking: MaskingStrT = None
- mthresh: float = None
- mask: dios.DictOfSeries = None
+ masking: MaskingStrT
+ mthresh: float
+ mask: dios.DictOfSeries
def register(masking: MaskingStrT = "all", module: Optional[str] = None):
@@ -50,9 +50,9 @@ def register(masking: MaskingStrT = "all", module: Optional[str] = None):
# nevertheless if it is called plain or via `SaQC.func`.
@wraps(func)
def callWrapper(*args, **kwargs):
- args, kwargs, ctrl = _preCall(func, args, kwargs, masking, func_name)
+ args, kwargs, old_state = _preCall(func, args, kwargs, masking, func_name)
result = func(*args, **kwargs)
- return _postCall(result, ctrl, func_name)
+ return _postCall(result, old_state)
FUNC_MAP[func_name] = SaQCFunction(func_name, callWrapper)
@@ -92,7 +92,7 @@ def _preCall(func: callable, args: tuple, kwargs: dict, masking: MaskingStrT, fn
arguments to be passed to the actual call
kwargs: dict
keyword-arguments to be passed to the actual call
- ctrl: CallCtrl
+ state: CallState
control keyword-arguments passed to `_postCall`
"""
@@ -100,23 +100,28 @@ def _preCall(func: callable, args: tuple, kwargs: dict, masking: MaskingStrT, fn
kwargs['to_mask'] = mthresh
data, field, flagger, *args = args
- ctrl = CallCtrl(func, data.copy(), field, flagger.copy(), args, kwargs, masking=masking, mthresh=mthresh)
# handle data - masking
columns = _getMaskingColumns(data, field, masking)
- data, mask = _maskData(data, flagger, columns, mthresh)
+ masked_data, mask = _maskData(data, flagger, columns, mthresh)
- # store mask
- ctrl.mask = mask
+ # store current state
+ state = CallState(
+ func=func,
+ data=data, flagger=flagger, field=field,
+ args=args, kwargs=kwargs,
+ masking=masking, mthresh=mthresh,
+ mask=mask
+ )
# handle flags - clearing
- flagger = _prepareFlags(flagger, masking)
+ prepped_flagger = _prepareFlags(flagger, masking)
- args = data, field, flagger, *args
- return args, kwargs, ctrl
+ args = masked_data, field, prepped_flagger, *args
+ return args, kwargs, state
-def _postCall(result, ctrl: CallCtrl, fname: str) -> FuncReturnT:
+def _postCall(result, old_state: CallState) -> FuncReturnT:
"""
Handler that runs after any call to a saqc-function.
@@ -128,19 +133,16 @@ def _postCall(result, ctrl: CallCtrl, fname: str) -> FuncReturnT:
result : tuple
the result from the called function, namely: data and flagger
- ctrl : dict
+ old_state : dict
control keywords from `_preCall`
- fname : str
- Name of the (just) called saqc-function
-
Returns
-------
data, flagger : dios.DictOfSeries, saqc.flagger.Flagger
"""
data, flagger = result
- flagger = _restoreFlags(flagger, ctrl)
- data = _unmaskData(data, ctrl)
+ flagger = _restoreFlags(flagger, old_state)
+ data = _unmaskData(data, old_state)
return data, flagger
@@ -162,7 +164,7 @@ def _getMaskingColumns(data: dios.DictOfSeries, field: str, masking: MaskingStrT
if masking == 'field':
return pd.Index([field])
- raise ValueError(f"wrong use of `register(masking={ctrl.masking})`")
+ raise ValueError(f"wrong use of `register(masking={masking})`")
def _getMaskingThresh(masking, kwargs, fname):
@@ -220,9 +222,18 @@ def _getMaskingThresh(masking, kwargs, fname):
# TODO: this is heavily undertested
def _maskData(data, flagger, columns, thresh) -> Tuple[dios.DictOfSeries, dios.DictOfSeries]:
"""
- Mask data with Nans by flags worse that a threshold and according to masking keyword in decorator.
+ Mask data with Nans by flags worse that a threshold and according to ``masking`` keyword
+ from the functions decorator.
+
+ Returns
+ -------
+ masked : dios.DictOfSeries
+ masked data, same dim as original
+ mask : dios.DictOfSeries
+ boolean dios of same dim as `masked`. True, where data was masked, elsewhere False.
"""
mask = dios.DictOfSeries(columns=columns)
+ data = data.copy()
# we use numpy here because it is faster
for c in columns:
@@ -250,38 +261,41 @@ def _getMask(flags: Union[np.array, pd.Series], thresh: float) -> Union[np.array
def _prepareFlags(flagger: Flagger, masking) -> Flagger:
"""
- Clear flags before each call.
+ Prepare flags before each call. Always returns a copy.
+
+ Currently this only clears the flags, but in future,
+ this should be sliced the flagger to the columns, that
+ the saqc-function needs.
"""
# Either the index or the columns itself changed
if masking == 'none':
- return flagger
+ return flagger.copy()
return initFlagsLike(flagger, initial_value=UNTOUCHED)
-def _restoreFlags(flagger: Flagger, ctrl: CallCtrl):
- if ctrl.masking == 'none':
+def _restoreFlags(flagger: Flagger, old_state: CallState):
+ if old_state.masking == 'none':
return flagger
- result = ctrl.flagger
-
columns = flagger.columns
# take field column and all possibly newly added columns
- if ctrl.masking == 'field':
- columns = columns.difference(ctrl.flagger.columns)
- columns = columns.append(pd.Index([ctrl.field]))
+ if old_state.masking == 'field':
+ columns = columns.difference(old_state.flagger.columns)
+ columns = columns.append(pd.Index([old_state.field]))
+ out = old_state.flagger
for c in columns:
# this implicitly squash the new-flagger history (RHS) to a single column, which than is appended to
# the old history (LHS). The new-flagger history possibly consist of multiple columns, one for each
# time flags was set to the flagger.
- result[c] = flagger[c]
+ out[c] = flagger[c]
- return result
+ return out
# TODO: this is heavily undertested
-def _unmaskData(data: dios.DictOfSeries, ctrl: CallCtrl) -> dios.DictOfSeries:
+def _unmaskData(data: dios.DictOfSeries, old_state: CallState) -> dios.DictOfSeries:
"""
Restore the masked data.
@@ -289,7 +303,7 @@ def _unmaskData(data: dios.DictOfSeries, ctrl: CallCtrl) -> dios.DictOfSeries:
-----
Even if this returns data, it work inplace !
"""
- if ctrl.masking == 'none':
+ if old_state.masking == 'none':
return data
# we have two options to implement this:
@@ -313,28 +327,27 @@ def _unmaskData(data: dios.DictOfSeries, ctrl: CallCtrl) -> dios.DictOfSeries:
# col in new only : new (keep column)
# col in old only : new (ignore, was deleted)
- old = ctrl # this alias simplifies reading a lot
- columns = old.mask.columns.intersection(data.columns) # in old, in masked, in new
+ columns = old_state.mask.columns.intersection(data.columns) # in old, in masked, in new
for c in columns:
# ignore
- if old.data[c].empty or data[c].empty or old.mask[c].empty:
+ if old_state.data[c].empty or data[c].empty or old_state.mask[c].empty:
continue
# on index changed, we simply ignore the old data
- if not old.data[c].index.equals(data[c].index):
+ if not old_state.data[c].index.equals(data[c].index):
continue
- restore_old_mask = old.mask[c].to_numpy() & data[c].isna().to_numpy()
+ restore_old_mask = old_state.mask[c].to_numpy() & data[c].isna().to_numpy()
# we have nothing to restore
if not any(restore_old_mask):
continue
# restore old values if no new are present
- v_old, v_new = old.data[c].to_numpy(), data[c].to_numpy()
- data.loc[:, c] = np.where(restore_old_mask, v_old, v_new)
+ old, new = old_state.data[c].to_numpy(), data[c].to_numpy()
+ data.loc[:, c] = np.where(restore_old_mask, old, new)
return data
--
GitLab
From 40ae6d53bc680bea76eeeb9a15aa9e86f92c4b7d Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 19 Mar 2021 21:16:24 +0100
Subject: [PATCH 057/180] removed inplace modification of History as it holds
to many implicit pitfalls.
---
saqc/flagger/flags.py | 6 +-----
saqc/funcs/tools.py | 1 +
2 files changed, 2 insertions(+), 5 deletions(-)
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index f5e36cc44..099e3c924 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -31,11 +31,7 @@ class _HistAccess:
self.obj = obj
def __getitem__(self, key: str) -> History:
- # we don't know, what the user wants. Although we're not
- # encouraging inplace modification of the history, the
- # user may do it, so we remove the cached column here.
- self.obj._cache.pop(key, None)
- return self.obj._data[key]
+ return self.obj._data[key].copy()
def __setitem__(self, key: str, value: Union[History, pd.DataFrame]):
if not isinstance(value, History):
diff --git a/saqc/funcs/tools.py b/saqc/funcs/tools.py
index edbb4c80c..c2a451b95 100644
--- a/saqc/funcs/tools.py
+++ b/saqc/funcs/tools.py
@@ -44,6 +44,7 @@ def copy(data: DictOfSeries, field: str, flagger: Flagger, new_field: str, **kwa
raise ValueError(f"{field}: field already exist")
data[new_field] = data[field].copy()
+ # implicit copy in history access
flagger.history[new_field] = flagger.history[field]
return data, flagger
--
GitLab
From deda7f7f3256f7cf78519f8990088798bd87b44a Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 19 Mar 2021 22:20:36 +0100
Subject: [PATCH 058/180] fixed todo-texts
---
saqc/funcs/generic.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index 4678bec62..1058da740 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -136,6 +136,7 @@ def process(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd
data[field] = _execGeneric(flagger, data, func, field, nodata).squeeze()
# TODO: the former comment wished to overwrite the column, but i'm not sure -- palmb
+ # see #GL177
if field in flagger:
flagger.drop(field)
@@ -146,6 +147,7 @@ def process(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd
@register(masking='all', module="generic")
def flag(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd.Series], pd.Series],
nodata: float = np.nan, flag=BAD, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+ # TODO : fix docstring, check if all still works
"""
a function to flag a data column by evaluation of a generic expression.
@@ -211,7 +213,6 @@ def flag(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd.Se
>>> lambda level: isflagged(level, flag=DOUBTFUL, comparator='>')
- # TODO : fix text
If you are unsure about the used flaggers flagging level names, you can use the reserved key words BAD, UNFLAGGED
and GOOD, to refer to the worst (BAD), best(GOOD) or unflagged (UNFLAGGED) flagging levels. For example.
--
GitLab
From 64670e1c6c56fb3386cf372dddea30de615efe8c Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Sat, 20 Mar 2021 01:28:39 +0100
Subject: [PATCH 059/180] fixed register bug
---
saqc/core/register.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 9bb841b49..50df11b6b 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -284,7 +284,7 @@ def _restoreFlags(flagger: Flagger, old_state: CallState):
columns = columns.difference(old_state.flagger.columns)
columns = columns.append(pd.Index([old_state.field]))
- out = old_state.flagger
+ out = old_state.flagger.copy()
for c in columns:
# this implicitly squash the new-flagger history (RHS) to a single column, which than is appended to
# the old history (LHS). The new-flagger history possibly consist of multiple columns, one for each
--
GitLab
From a0147299b6c29943950ca538b8dce396358a46f6 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Sat, 20 Mar 2021 02:41:03 +0100
Subject: [PATCH 060/180] fixed harmo tests
---
tests/funcs/test_harm_funcs.py | 273 +++++++++++++--------------------
1 file changed, 103 insertions(+), 170 deletions(-)
diff --git a/tests/funcs/test_harm_funcs.py b/tests/funcs/test_harm_funcs.py
index 0675b3aeb..f78f8e573 100644
--- a/tests/funcs/test_harm_funcs.py
+++ b/tests/funcs/test_harm_funcs.py
@@ -1,17 +1,13 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-
-# see test/functs/conftest.py for global fixtures "course_..."
import pytest
import numpy as np
import pandas as pd
import dios
-from test.common import TESTFLAGGER
from saqc.flagger import Flagger, initFlagsLike
-from saqc.common import BAD
-
+from saqc.constants import BAD, UNFLAGGED
from saqc.funcs.resampling import (
linear,
interpolate,
@@ -20,10 +16,6 @@ from saqc.funcs.resampling import (
mapToOriginal,
)
-RESHAPERS = ["nshift", "fshift", "bshift", "nagg", "bagg", "fagg", "interpolation"]
-
-INTERPOLATIONS = ["time", "polynomial"]
-
@pytest.fixture
def data():
@@ -41,184 +33,125 @@ def data():
return data
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-@pytest.mark.parametrize("reshaper", RESHAPERS)
-def test_harmSingleVarIntermediateFlagging(data, flagger, reshaper):
+@pytest.mark.parametrize("reshaper", ["nshift", "fshift", "bshift", "nagg", "bagg", "fagg", "interpolation"])
+def test_harmSingleVarIntermediateFlagging(data, reshaper):
flagger = initFlagsLike(data)
- # make pre harm copies:
+ field = 'data'
+
pre_data = data.copy()
- pre_flags = flagger['data']
- freq = "15min"
- assert len(data.columns) == 1
- field = data.columns[0]
- data, flagger = linear(data, "data", flagger, freq)
+ pre_flagger = flagger.copy()
+
+ data, flagger = linear(data, field, flagger, freq="15min")
+
# flag something bad
- f_ser = pd.Series(data=[-np.inf] * len(data[field]), index=data[field].index)
- f_ser[3:4] = BAD
- flagger[field] = f_ser
- data, flagger = mapToOriginal(data, "data", flagger, method="inverse_" + reshaper)
- d = data[field]
- if reshaper == "nagg":
- assert flagger.isFlagged(loc=d.index[3:7]).squeeze().all()
- assert (~flagger.isFlagged(loc=d.index[0:3]).squeeze()).all()
- assert (~flagger.isFlagged(loc=d.index[7:]).squeeze()).all()
- if reshaper == "nshift":
- assert (flagger.isFlagged().squeeze() == [False, False, False, False, True, False, False, False, False]).all()
- if reshaper == "bagg":
- assert flagger.isFlagged(loc=d.index[5:7]).squeeze().all()
- assert (~flagger.isFlagged(loc=d.index[0:5]).squeeze()).all()
- assert (~flagger.isFlagged(loc=d.index[7:]).squeeze()).all()
- if reshaper == "bshift":
- assert (flagger.isFlagged().squeeze() == [False, False, False, False, False, True, False, False, False]).all()
- if reshaper == "fagg":
- assert flagger.isFlagged(loc=d.index[3:5]).squeeze().all()
- assert (~flagger.isFlagged(loc=d.index[0:3]).squeeze()).all()
- assert (~flagger.isFlagged(loc=d.index[5:]).squeeze()).all()
- if reshaper == "fshift":
- assert (flagger.isFlagged().squeeze() == [False, False, False, False, True, False, False, False, False]).all()
-
- flags = flagger.getFlags()
- assert pre_data[field].equals(data[field])
- assert len(data[field]) == len(flags[field])
- assert (pre_flags[field].index == flags[field].index).all()
-
-
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_harmSingleVarInterpolations(data, flagger):
- flagger = flagger.initFlags(data)
- field = data.columns[0]
- pre_data = data[field]
- pre_flags = flagger.getFlags(field)
- tests = [
- (
- "nagg",
- "15Min",
- pd.Series(
- data=[-87.5, -25.0, 0.0, 37.5, 50.0],
- index=pd.date_range("2011-01-01 00:00:00", "2011-01-01 01:00:00", freq="15min"),
- ),
- ),
- (
- "nagg",
- "30Min",
- pd.Series(
- data=[-87.5, -25.0, 87.5],
- index=pd.date_range("2011-01-01 00:00:00", "2011-01-01 01:00:00", freq="30min"),
- ),
- ),
- (
- "bagg",
- "15Min",
- pd.Series(
- data=[-50.0, -37.5, -37.5, 12.5, 37.5, 50.0],
- index=pd.date_range("2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15min"),
- ),
- ),
- (
- "bagg",
- "30Min",
- pd.Series(
- data=[-50.0, -75.0, 50.0, 50.0],
- index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30min"),
- ),
- ),
- ]
-
- for interpolation, freq, expected in tests:
- data_harm, flagger_harm = aggregate(
- data, field, flagger, freq, value_func=np.sum, method=interpolation
- )
- assert data_harm[field].equals(expected)
- data_deharm, flagger_deharm = mapToOriginal(
- data_harm, "data", flagger_harm, method="inverse_" + interpolation
- )
- assert data_deharm[field].equals(pre_data)
- assert flagger_deharm.getFlags([field]).squeeze().equals(pre_flags)
-
- tests = [
- (
- "fshift",
- "15Min",
- pd.Series(
- data=[np.nan, -37.5, -25.0, 0.0, 37.5, 50.0],
- index=pd.date_range("2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15Min"),
- ),
- ),
- (
- "fshift",
- "30Min",
- pd.Series(
- data=[np.nan, -37.5, 0.0, 50.0],
- index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"),
- ),
- ),
- (
- "bshift",
- "15Min",
- pd.Series(
- data=[-50.0, -37.5, -25.0, 12.5, 37.5, 50.0],
- index=pd.date_range("2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15Min"),
- ),
- ),
- (
- "bshift",
- "30Min",
- pd.Series(
- data=[-50.0, -37.5, 12.5, 50.0],
- index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"),
- ),
- ),
- (
- "nshift",
- "15min",
- pd.Series(
- data=[np.nan, -37.5, -25.0, 12.5, 37.5, 50.0],
- index=pd.date_range("2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15Min"),
- ),
- ),
- (
- "nshift",
- "30min",
- pd.Series(
- data=[np.nan, -37.5, 12.5, 50.0],
- index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"),
- ),
- ),
- ]
-
- for interpolation, freq, expected in tests:
- data_harm, flagger_harm = shift(data, field, flagger, freq, method=interpolation)
- assert data_harm[field].equals(expected)
- data_deharm, flagger_deharm = mapToOriginal(
- data_harm, "data", flagger_harm, method="inverse_" + interpolation
- )
- assert data_deharm[field].equals(pre_data)
- assert flagger_deharm.getFlags([field]).squeeze().equals(pre_flags)
-
-
-@pytest.mark.parametrize("method", INTERPOLATIONS)
+ flagger[data[field].index[3:4], field] = BAD
+ data, flagger = mapToOriginal(data, field, flagger, method="inverse_" + reshaper)
+
+ assert len(data[field]) == len(flagger[field])
+ assert data[field].equals(pre_data[field])
+ assert flagger[field].index.equals(pre_flagger[field].index)
+
+ if 'agg' in reshaper:
+ if reshaper == "nagg":
+ start, end = 3, 7
+ elif reshaper == "fagg":
+ start, end = 3, 5
+ elif reshaper == "bagg":
+ start, end = 5, 7
+ else:
+ raise NotImplementedError('untested test case')
+
+ assert all(flagger[field].iloc[start:end])
+ assert all(~flagger[field].iloc[:start])
+ assert all(~flagger[field].iloc[end:])
+
+ elif 'shift' in reshaper:
+ if reshaper == "nshift":
+ exp = [False, False, False, False, True, False, False, False, False]
+ elif reshaper == "fshift":
+ exp = [False, False, False, False, True, False, False, False, False]
+ elif reshaper == "bshift":
+ exp = [False, False, False, False, False, True, False, False, False]
+ else:
+ raise NotImplementedError('untested test case')
+
+ flagged = flagger[field] > UNFLAGGED
+ assert all(flagged == exp)
+
+ else:
+ raise NotImplementedError('untested test case')
+
+
+@pytest.mark.parametrize(
+ 'params, expected',
+ [
+ (("nagg", "15Min"), pd.Series(data=[-87.5, -25.0, 0.0, 37.5, 50.0], index=pd.date_range("2011-01-01 00:00:00", "2011-01-01 01:00:00", freq="15min"))),
+ (("nagg", "30Min"), pd.Series(data=[-87.5, -25.0, 87.5], index=pd.date_range("2011-01-01 00:00:00", "2011-01-01 01:00:00", freq="30min"))),
+ (("bagg", "15Min"), pd.Series(data=[-50.0, -37.5, -37.5, 12.5, 37.5, 50.0], index=pd.date_range("2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15min"))),
+ (("bagg", "30Min"), pd.Series(data=[-50.0, -75.0, 50.0, 50.0], index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30min"))),
+ ])
+def test_harmSingleVarInterpolationAgg(data, params, expected):
+ flagger = initFlagsLike(data)
+ field = 'data'
+ pre_data = data.copy()
+ pre_flaggger = flagger.copy()
+ method, freq = params
+
+ data_harm, flagger_harm = aggregate(data, field, flagger, freq, value_func=np.sum, method=method)
+ assert data_harm[field].equals(expected)
+
+ data_deharm, flagger_deharm = mapToOriginal(data_harm, "data", flagger_harm, method="inverse_" + method)
+ assert data_deharm[field].equals(pre_data[field])
+ assert flagger_deharm[field].equals(pre_flaggger[field])
+
+
+@pytest.mark.parametrize(
+ 'params, expected',
+ [
+ (("fshift", "15Min"), pd.Series(data=[np.nan, -37.5, -25.0, 0.0, 37.5, 50.0], index=pd.date_range("2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15Min"))),
+ (("fshift", "30Min"), pd.Series(data=[np.nan, -37.5, 0.0, 50.0], index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"))),
+ (("bshift", "15Min"), pd.Series(data=[-50.0, -37.5, -25.0, 12.5, 37.5, 50.0], index=pd.date_range("2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15Min"))),
+ (("bshift", "30Min"), pd.Series(data=[-50.0, -37.5, 12.5, 50.0], index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"))),
+ (("nshift", "15min"), pd.Series(data=[np.nan, -37.5, -25.0, 12.5, 37.5, 50.0], index=pd.date_range("2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15Min"))),
+ (("nshift", "30min"), pd.Series(data=[np.nan, -37.5, 12.5, 50.0], index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"))),
+ ])
+def test_harmSingleVarInterpolationShift(data, params, expected):
+ flagger = initFlagsLike(data)
+ field = 'data'
+ pre_data = data.copy()
+ pre_flagger = flagger.copy()
+ method, freq = params
+
+ data_harm, flagger_harm = shift(data, field, flagger, freq, method=method)
+ assert data_harm[field].equals(expected)
+
+ data_deharm, flagger_deharm = mapToOriginal(data_harm, "data", flagger_harm, method="inverse_" + method)
+ assert data_deharm[field].equals(pre_data[field])
+ assert flagger_deharm[field].equals(pre_flagger[field])
+
+
+@pytest.mark.parametrize("method", ["time", "polynomial"])
def test_gridInterpolation(data, method):
freq = "15min"
- data = data.squeeze()
- field = data.name
+ field = 'data'
+ data = data[field]
data = (data * np.sin(data)).append(data.shift(1, "2h")).shift(1, "3s")
data = dios.DictOfSeries(data)
- flagger = TESTFLAGGER[0].initFlags(data)
+ flagger = initFlagsLike(data)
# we are just testing if the interpolation gets passed to the series without causing an error:
-
interpolate(data, field, flagger, freq, method=method, downcast_interpolation=True)
+
if method == "polynomial":
interpolate(data, field, flagger, freq, order=2, method=method, downcast_interpolation=True)
interpolate(data, field, flagger, freq, order=10, method=method, downcast_interpolation=True)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_wrapper(data, flagger):
+def test_wrapper(data):
# we are only testing, whether the wrappers do pass processing:
- field = data.columns[0]
+ field = 'data'
freq = "15min"
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
linear(data, field, flagger, freq, to_drop=None)
aggregate(data, field, flagger, freq, value_func=np.nansum, method="nagg", to_drop=None)
--
GitLab
From 8adf6744e8a501ac6c0127f5af98d07153a15d5b Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Sat, 20 Mar 2021 02:59:02 +0100
Subject: [PATCH 061/180] fixed other processing tests
---
saqc/flagger/flags.py | 4 ++--
saqc/funcs/drift.py | 3 ++-
tests/funcs/test_modelling.py | 29 +++++++++++--------------
tests/funcs/test_proc_functions.py | 34 ++++++++++++------------------
4 files changed, 30 insertions(+), 40 deletions(-)
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index e6ef0ad5b..e6d3076fa 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -401,8 +401,8 @@ def mergeHistoryByFunc(flags: Flags, field, source, merge_func, merge_func_kws,
target_history = flags.history[field]
source_history = flags.history[source]
new_target_history = History()
- import pdb
- pdb.set_trace()
+ # import pdb
+ # pdb.set_trace()
for k in target_history.hist.columns:
col_args_h = dict(source_col=source_history.hist[k])
col_args_m = dict(source_col=source_history.mask[k])
diff --git a/saqc/funcs/drift.py b/saqc/funcs/drift.py
index dc2265e8a..ea1a4d20c 100644
--- a/saqc/funcs/drift.py
+++ b/saqc/funcs/drift.py
@@ -454,7 +454,8 @@ def correctRegimeAnomaly(
cluster_field: ColumnName,
model: CurveFitter,
regime_transmission: Optional[FreqString]=None,
- x_date: bool=False
+ x_date: bool=False,
+ **kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
Function fits the passed model to the different regimes in data[field] and tries to correct
diff --git a/tests/funcs/test_modelling.py b/tests/funcs/test_modelling.py
index 6d99d5786..23cc82ab2 100644
--- a/tests/funcs/test_modelling.py
+++ b/tests/funcs/test_modelling.py
@@ -6,23 +6,20 @@
import dios
+from saqc.flagger import initFlagsLike
from saqc.funcs.tools import mask
from saqc.funcs.residues import calculatePolynomialResidues, calculateRollingResidues
from tests.fixtures import *
-from tests.common import TESTFLAGGER
-TF = TESTFLAGGER[:1]
-
-@pytest.mark.parametrize("flagger", TF)
@pytest.mark.parametrize("dat", [pytest.lazy_fixture("course_2")])
-def test_modelling_polyFit_forRegular(dat, flagger):
+def test_modelling_polyFit_forRegular(dat):
data, _ = dat(freq="10min", periods=30, initial_level=0, final_level=100, out_val=-100)
# add some nice sine distortion
data = data + 10 * np.sin(np.arange(0, len(data.indexes[0])))
data = dios.DictOfSeries(data)
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
result1, _ = calculatePolynomialResidues(data, "data", flagger, 11, 2, numba=False)
result2, _ = calculatePolynomialResidues(data, "data", flagger, 11, 2, numba=True)
assert (result1["data"] - result2["data"]).abs().max() < 10 ** -10
@@ -35,39 +32,37 @@ def test_modelling_polyFit_forRegular(dat, flagger):
assert result5["data"].iloc[10:19].isna().all()
-@pytest.mark.parametrize("flagger", TF)
@pytest.mark.parametrize("dat", [pytest.lazy_fixture("course_2")])
-def test_modelling_rollingMean_forRegular(dat, flagger):
+def test_modelling_rollingMean_forRegular(dat):
data, _ = dat(freq="10min", periods=30, initial_level=0, final_level=100, out_val=-100)
data = dios.DictOfSeries(data)
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
calculateRollingResidues(data, "data", flagger, 5, func=np.mean, eval_flags=True, min_periods=0, center=True)
calculateRollingResidues(data, "data", flagger, 5, func=np.mean, eval_flags=True, min_periods=0, center=False)
-@pytest.mark.parametrize("flagger", TF)
@pytest.mark.parametrize("dat", [pytest.lazy_fixture("course_1")])
-def test_modelling_mask(dat, flagger):
+def test_modelling_mask(dat):
data, _ = dat()
data = dios.DictOfSeries(data)
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
data_seasonal, flagger_seasonal = mask(data, "data", flagger, mode='periodic', period_start="20:00",
period_end="40:00", include_bounds=False)
- flaggs = flagger_seasonal._flags["data"]
+ flaggs = flagger_seasonal["data"]
assert flaggs[np.logical_and(20 <= flaggs.index.minute, 40 >= flaggs.index.minute)].isna().all()
data_seasonal, flagger_seasonal = mask(data, "data", flagger, mode='periodic', period_start="15:00:00",
period_end="02:00:00")
- flaggs = flagger_seasonal._flags["data"]
+ flaggs = flagger_seasonal["data"]
assert flaggs[np.logical_and(15 <= flaggs.index.hour, 2 >= flaggs.index.hour)].isna().all()
data_seasonal, flagger_seasonal = mask(data, "data", flagger, mode='periodic', period_start="03T00:00:00",
period_end="10T00:00:00")
- flaggs = flagger_seasonal._flags["data"]
+ flaggs = flagger_seasonal["data"]
assert flaggs[np.logical_and(3 <= flaggs.index.hour, 10 >= flaggs.index.hour)].isna().all()
mask_ser = pd.Series(False, index=data["data"].index)
mask_ser[::5] = True
data["mask_ser"] = mask_ser
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
data_masked, flagger_masked = mask(data, "data", flagger, mode='mask_var', mask_var="mask_ser")
- flaggs = flagger_masked._flags["data"]
+ flaggs = flagger_masked["data"]
assert flaggs[data_masked['mask_ser']].isna().all()
diff --git a/tests/funcs/test_proc_functions.py b/tests/funcs/test_proc_functions.py
index d7cada078..d9d137359 100644
--- a/tests/funcs/test_proc_functions.py
+++ b/tests/funcs/test_proc_functions.py
@@ -7,6 +7,7 @@
import dios
from saqc.constants import *
+from saqc.flagger import initFlagsLike
from saqc.funcs.transformation import transform
from saqc.funcs.drift import correctOffset
from saqc.funcs.interpolation import interpolateByRolling, interpolateInvalid, interpolateIndex
@@ -14,15 +15,13 @@ from saqc.funcs.resampling import resample
from saqc.lib.ts_operators import linearInterpolation, polynomialInterpolation
from tests.fixtures import *
-from tests.common import TESTFLAGGER
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_rollingInterpolateMissing(course_5, flagger):
+def test_rollingInterpolateMissing(course_5):
data, characteristics = course_5(periods=10, nan_slice=[5, 6])
field = data.columns[0]
data = dios.DictOfSeries(data)
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
dataInt, *_ = interpolateByRolling(
data, field, flagger, 3, func=np.median, center=True, min_periods=0, interpol_flag=UNFLAGGED
)
@@ -35,12 +34,11 @@ def test_rollingInterpolateMissing(course_5, flagger):
assert dataInt[field][characteristics["missing"]].isna().all()
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_interpolateMissing(course_5, flagger):
+def test_interpolateMissing(course_5):
data, characteristics = course_5(periods=10, nan_slice=[5])
field = data.columns[0]
data = dios.DictOfSeries(data)
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
dataLin, *_ = interpolateInvalid(data, field, flagger, method="linear")
dataPoly, *_ = interpolateInvalid(data, field, flagger, method="polynomial")
assert dataLin[field][characteristics["missing"]].notna().all()
@@ -54,12 +52,11 @@ def test_interpolateMissing(course_5, flagger):
assert dataLin3[field][characteristics["missing"]].notna().all()
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_transform(course_5, flagger):
+def test_transform(course_5):
data, characteristics = course_5(periods=10, nan_slice=[5, 6])
field = data.columns[0]
data = dios.DictOfSeries(data)
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
data1, *_ = transform(data, field, flagger, func=linearInterpolation)
assert data1[field][characteristics["missing"]].isna().all()
data1, *_ = transform(data, field, flagger, func=lambda x: linearInterpolation(x, inter_limit=3))
@@ -70,35 +67,32 @@ def test_transform(course_5, flagger):
assert data1[field][characteristics["missing"]].notna().all()
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_resample(course_5, flagger):
+def test_resample(course_5):
data, characteristics = course_5(freq="1min", periods=30, nan_slice=[1, 11, 12, 22, 24, 26])
field = data.columns[0]
data = dios.DictOfSeries(data)
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
data1, *_ = resample(data, field, flagger, "10min", np.mean, max_invalid_total_d=2, max_invalid_consec_d=1)
assert ~np.isnan(data1[field].iloc[0])
assert np.isnan(data1[field].iloc[1])
assert np.isnan(data1[field].iloc[2])
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_interpolateGrid(course_5, course_3, flagger):
+def test_interpolateGrid(course_5, course_3):
data, _ = course_5()
data_grid, characteristics = course_3()
data['grid'] = data_grid.to_df()
# data = dios.DictOfSeries(data)
- flagger = flagger.initFlags(data)
+ flagger = initFlagsLike(data)
dataInt, *_ = interpolateIndex(data, 'data', flagger, '1h', 'time', grid_field='grid', inter_limit=10)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_offsetCorrecture(flagger):
+def test_offsetCorrecture():
data = pd.Series(0, index=pd.date_range('2000', freq='1d', periods=100), name='dat')
data.iloc[30:40] = -100
data.iloc[70:80] = 100
data = dios.DictOfSeries(data)
- flagger = flagger.initFlags(data)
- data, flagger = correctOffset(data, 'dat', flagger, 40, 20, '3d', 1)
+ flagger = initFlagsLike(data)
+ data, _ = correctOffset(data, 'dat', flagger, 40, 20, '3d', 1)
assert (data == 0).all()[0]
--
GitLab
From e02c8d846d21192b2fc3cf65c1d909d41b4f3e26 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Sat, 20 Mar 2021 15:13:04 +0100
Subject: [PATCH 062/180] cleanup, and to_mask-fix
---
saqc/core/register.py | 4 +--
saqc/funcs/interpolation.py | 53 ++++++++++++++++------------------
saqc/funcs/resampling.py | 7 ++---
saqc/lib/ts_operators.py | 2 +-
tests/funcs/test_harm_funcs.py | 9 ++++--
5 files changed, 37 insertions(+), 38 deletions(-)
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 50df11b6b..ce88dc4bc 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -237,7 +237,7 @@ def _maskData(data, flagger, columns, thresh) -> Tuple[dios.DictOfSeries, dios.D
# we use numpy here because it is faster
for c in columns:
- col_mask = _getMask(flagger[c].to_numpy(), thresh)
+ col_mask = isflagged(flagger[c].to_numpy(), thresh)
if any(col_mask):
col_data = data[c].to_numpy(dtype=np.float64)
@@ -249,7 +249,7 @@ def _maskData(data, flagger, columns, thresh) -> Tuple[dios.DictOfSeries, dios.D
return data, mask
-def _getMask(flags: Union[np.array, pd.Series], thresh: float) -> Union[np.array, pd.Series]:
+def isflagged(flags: Union[np.array, pd.Series], thresh: float) -> Union[np.array, pd.Series]:
"""
Return a mask of flags accordingly to `thresh`. Return type is same as flags.
"""
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index dd5036d9c..c0b9b8ee0 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -10,21 +10,26 @@ import pandas as pd
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core.register import register
+from saqc.core.register import register, isflagged
from saqc.flagger import Flagger
from saqc.flagger.flags import applyFunctionOnHistory
from saqc.lib.tools import toSequence, evalFreqStr, getDropMask
from saqc.lib.ts_operators import interpolateNANs
+_SUPPORTED_METHODS = Literal[
+ "linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric",
+ "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"
+]
+
@register(masking='field', module="interpolation")
def interpolateByRolling(
data: DictOfSeries, field: str, flagger: Flagger,
winsz: Union[str, int],
- func: Callable[[pd.Series], float]=np.median,
- center: bool=True,
- min_periods: int=0,
+ func: Callable[[pd.Series], float] = np.median,
+ center: bool = True,
+ min_periods: int = 0,
flag: float = UNFLAGGED,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
@@ -93,10 +98,10 @@ def interpolateInvalid(
data: DictOfSeries,
field: str,
flagger: Flagger,
- method: Literal["linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric", "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"],
- inter_order: int=2,
- inter_limit: int=2,
- downgrade_interpolation: bool=False,
+ method: _SUPPORTED_METHODS,
+ inter_order: int = 2,
+ inter_limit: int = 2,
+ downgrade_interpolation: bool = False,
not_interpol_flags=None,
flag: float = UNFLAGGED,
**kwargs
@@ -165,7 +170,7 @@ def interpolateInvalid(
return data, flagger
-def _overlap_rs(x, freq='1min', fill_value=-np.inf):
+def _overlap_rs(x, freq='1min', fill_value=UNFLAGGED):
end = x.index[-1].ceil(freq)
x = x.resample(freq).max()
x = x.combine(x.shift(1, fill_value=fill_value), max)
@@ -184,10 +189,7 @@ def interpolateIndex(
field: str,
flagger: Flagger,
freq: str,
- method: Literal[
- "linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric",
- "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"
- ],
+ method: _SUPPORTED_METHODS,
inter_order: int = 2,
downgrade_interpolation: bool = False,
inter_limit: int = 2,
@@ -252,23 +254,19 @@ def interpolateIndex(
start, end = datcol.index[0].floor(freq), datcol.index[-1].ceil(freq)
grid_index = pd.date_range(start=start, end=end, freq=freq, name=datcol.index.name)
- # always injected by register
- to_mask = kwargs['to_mask']
+ flagged = isflagged(flagscol, kwargs['to_mask'])
- datcol.drop(flagscol[flagscol >= to_mask].index, inplace=True)
- datcol.dropna(inplace=True)
- dat_index = datcol.index
+ # drop all points that hold no relevant grid information
+ datcol = datcol[~flagged].dropna()
# account for annoying case of subsequent frequency aligned values,
# that differ exactly by the margin of 2*freq
- gaps = ((dat_index[1:] - dat_index[:-1]) == 2*pd.Timedelta(freq))
- gaps = dat_index[1:][gaps]
- aligned_gaps = gaps.join(grid_index, how='inner')
- if not aligned_gaps.empty:
- aligned_gaps = aligned_gaps.shift(-1, freq)
+ gaps = datcol.index[1:] - datcol.index[:-1] == 2 * pd.Timedelta(freq)
+ gaps = datcol.index[1:][gaps]
+ gaps = gaps.intersection(grid_index).shift(-1, freq)
# prepare grid interpolation:
- datcol = datcol.reindex(datcol.index.join(grid_index, how="outer",))
+ datcol = datcol.reindex(datcol.index.union(grid_index))
# do the grid interpolation
inter_data = interpolateNANs(
@@ -280,18 +278,17 @@ def interpolateIndex(
)
# override falsely interpolated values:
- inter_data[aligned_gaps] = np.nan
+ inter_data[gaps] = np.nan
# store interpolated grid
data[field] = inter_data[grid_index]
# flags reshaping
- flagscol.drop(flagscol[flagscol >= to_mask].index, inplace=True)
+ flagscol = flagscol[~flagged]
flagscol = _overlap_rs(flagscol, freq, UNFLAGGED)
flagger = applyFunctionOnHistory(
- flagger,
- field,
+ flagger, field,
hist_func=_overlap_rs, hist_kws=dict(freq=freq, fill_value=UNFLAGGED),
mask_func=_overlap_rs, mask_kws=dict(freq=freq, fill_value=False),
last_column=flagscol
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 3e24cd505..848cf6ee9 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -12,7 +12,7 @@ import pandas as pd
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core.register import register
+from saqc.core.register import register, isflagged
from saqc.flagger import Flagger, initFlagsLike, History
from saqc.funcs.tools import copy, drop, rename
from saqc.funcs.interpolation import interpolateIndex
@@ -329,7 +329,7 @@ def mapToOriginal(
"""
newfield = str(field) + '_original'
- data, flagger = reindexFlags(data, newfield, flagger, method, source=field, to_drop=to_drop, **kwargs)
+ data, flagger = reindexFlags(data, newfield, flagger, method, source=field, to_mask=False)
data, flagger = drop(data, field, flagger)
data, flagger = rename(data, newfield, flagger, field)
return data, flagger
@@ -756,8 +756,7 @@ def reindexFlags(
merge_dict = dict(freq=tolerance, method=projection_method)
if method[-5:] == "shift":
- to_mask = kwargs['to_mask']
- drop_mask = (target_datcol.isna() | target_flagscol >= to_mask)
+ drop_mask = (target_datcol.isna() | isflagged(target_flagscol, kwargs['to_mask']))
projection_method = METHOD2ARGS[method][0]
tolerance = METHOD2ARGS[method][1](freq)
merge_func = _inverseShift
diff --git a/saqc/lib/ts_operators.py b/saqc/lib/ts_operators.py
index 44f91cb64..de9de79d2 100644
--- a/saqc/lib/ts_operators.py
+++ b/saqc/lib/ts_operators.py
@@ -203,7 +203,7 @@ def interpolateNANs(data, method, order=2, inter_limit=2, downgrade_interpolatio
"""
inter_limit = int(inter_limit)
data = pd.Series(data).copy()
- gap_mask = (data.rolling(inter_limit, min_periods=0).apply(lambda x: np.sum(np.isnan(x)), raw=True)) != inter_limit
+ gap_mask = data.isna().rolling(inter_limit, min_periods=0).sum() != inter_limit
if inter_limit == 2:
gap_mask = gap_mask & gap_mask.shift(-1, fill_value=True)
diff --git a/tests/funcs/test_harm_funcs.py b/tests/funcs/test_harm_funcs.py
index f78f8e573..a83368090 100644
--- a/tests/funcs/test_harm_funcs.py
+++ b/tests/funcs/test_harm_funcs.py
@@ -61,9 +61,9 @@ def test_harmSingleVarIntermediateFlagging(data, reshaper):
else:
raise NotImplementedError('untested test case')
- assert all(flagger[field].iloc[start:end])
- assert all(~flagger[field].iloc[:start])
- assert all(~flagger[field].iloc[end:])
+ assert all(flagger[field].iloc[start:end] > UNFLAGGED)
+ assert all(~flagger[field].iloc[:start] == UNFLAGGED)
+ assert all(~flagger[field].iloc[end:] == UNFLAGGED)
elif 'shift' in reshaper:
if reshaper == "nshift":
@@ -78,6 +78,9 @@ def test_harmSingleVarIntermediateFlagging(data, reshaper):
flagged = flagger[field] > UNFLAGGED
assert all(flagged == exp)
+ elif reshaper == 'interpolation':
+ pytest.skip('no testcase for interpolation')
+
else:
raise NotImplementedError('untested test case')
--
GitLab
From 81c842a16770e835673eff2aed9f44bc04191723 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Sat, 20 Mar 2021 15:55:48 +0100
Subject: [PATCH 063/180] introduced isflagged in register
---
saqc/core/register.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 50df11b6b..ce88dc4bc 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -237,7 +237,7 @@ def _maskData(data, flagger, columns, thresh) -> Tuple[dios.DictOfSeries, dios.D
# we use numpy here because it is faster
for c in columns:
- col_mask = _getMask(flagger[c].to_numpy(), thresh)
+ col_mask = isflagged(flagger[c].to_numpy(), thresh)
if any(col_mask):
col_data = data[c].to_numpy(dtype=np.float64)
@@ -249,7 +249,7 @@ def _maskData(data, flagger, columns, thresh) -> Tuple[dios.DictOfSeries, dios.D
return data, mask
-def _getMask(flags: Union[np.array, pd.Series], thresh: float) -> Union[np.array, pd.Series]:
+def isflagged(flags: Union[np.array, pd.Series], thresh: float) -> Union[np.array, pd.Series]:
"""
Return a mask of flags accordingly to `thresh`. Return type is same as flags.
"""
--
GitLab
From fe30a3d827ecf886b56823022be1f31c31476017 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Mon, 22 Mar 2021 13:24:48 +0100
Subject: [PATCH 064/180] fixed wrapper-test
---
tests/funcs/test_harm_funcs.py | 16 +++++++++++-----
1 file changed, 11 insertions(+), 5 deletions(-)
diff --git a/tests/funcs/test_harm_funcs.py b/tests/funcs/test_harm_funcs.py
index a83368090..3b1ec42c6 100644
--- a/tests/funcs/test_harm_funcs.py
+++ b/tests/funcs/test_harm_funcs.py
@@ -150,13 +150,19 @@ def test_gridInterpolation(data, method):
interpolate(data, field, flagger, freq, order=10, method=method, downcast_interpolation=True)
-def test_wrapper(data):
+@pytest.mark.parametrize('func, kws', [
+ ('linear', dict(to_drop=None)),
+ ('shift', dict(method="nshift", to_drop=None)),
+ ('interpolate', dict(method="spline")),
+ ('aggregate', dict(value_func=np.nansum, method="nagg", to_drop=None)),
+])
+def test_wrapper(data, func, kws):
# we are only testing, whether the wrappers do pass processing:
field = 'data'
freq = "15min"
flagger = initFlagsLike(data)
- linear(data, field, flagger, freq, to_drop=None)
- aggregate(data, field, flagger, freq, value_func=np.nansum, method="nagg", to_drop=None)
- shift(data, field, flagger, freq, method="nshift", to_drop=None)
- interpolate(data, field, flagger, freq, method="spline")
+ import saqc
+ func = getattr(saqc.funcs, func)
+ func(data, field, flagger, freq, **kws)
+
--
GitLab
From 2e2eb3cabf2db5f00c64bb9cb75e215844e7e127 Mon Sep 17 00:00:00 2001
From: Peter Luenenschloss <peter.luenenschloss@ufz.de>
Date: Mon, 22 Mar 2021 16:35:52 +0100
Subject: [PATCH 065/180] inverse interpolation running
---
saqc/flagger/flags.py | 31 +++++++------------------------
saqc/funcs/resampling.py | 26 +++++++++++++-------------
2 files changed, 20 insertions(+), 37 deletions(-)
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index e6d3076fa..c1dcb1ed6 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -376,12 +376,9 @@ def applyFunctionOnHistory(flags: Flags, column, hist_func, hist_kws, mask_func,
return flags
-def mergeHistoryByFunc(flags: Flags, field, source, merge_func, merge_func_kws, last_column=None):
+def appendHistory(flags: Flags, column, append_hist):
"""
- Merges the information of one history (source) into the other (field). (Without altering fields indices)
-
- Field indices remain unchanged. The merge is performed, via manipulating the field history values
- column wise according to `merge_func`.
+ Function, specialized for used in deharm context.
Parameters
@@ -398,25 +395,11 @@ def mergeHistoryByFunc(flags: Flags, field, source, merge_func, merge_func_kws,
"""
flags = flags.copy()
- target_history = flags.history[field]
- source_history = flags.history[source]
- new_target_history = History()
- # import pdb
- # pdb.set_trace()
- for k in target_history.hist.columns:
- col_args_h = dict(source_col=source_history.hist[k])
- col_args_m = dict(source_col=source_history.mask[k])
- col_args_h.update(merge_func_kws)
- col_args_m.update(merge_func_kws)
- new_target_history.hist[k] = merge_func(target_history.hist[k], **col_args_h)
- new_target_history.mask[k] = merge_func(target_history.mask[k], **col_args_m)
-
- if last_column is None:
- new_target_history.mask.iloc[:, -1:] = True
- else:
- new_target_history.append(last_column, force=True)
-
- flags.history[field] = new_target_history
+ new_history = flags.history[column]
+ for app_k in [k for k in append_hist.columns if k not in new_history.columns]:
+ new_history.hist[app_k] = append_hist.hist[app_k]
+ new_history.mask[app_k] = append_hist.mask[app_k]
+ flags.history[column] = new_history
return flags
# for now we keep this name
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 848cf6ee9..decac74df 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -18,7 +18,7 @@ from saqc.funcs.tools import copy, drop, rename
from saqc.funcs.interpolation import interpolateIndex
from saqc.lib.tools import getDropMask, evalFreqStr, getFreqDelta
from saqc.lib.ts_operators import shift2Freq, aggregate2Freq
-from saqc.flagger.flags import applyFunctionOnHistory, mergeHistoryByFunc
+from saqc.flagger.flags import applyFunctionOnHistory, appendHistory
from saqc.lib.rolling import customRoller
logger = logging.getLogger("SaQC")
@@ -621,16 +621,13 @@ def _getChunkBounds(target_datcol, flagscol, freq):
return ignore_flags
-def _inverseInterpolation(target_flagscol, source_col=None, freq=None, chunk_bounds=None):
+def _inverseInterpolation(source_col, freq=None, chunk_bounds=None, target_flagscol=None):
source_col = source_col.copy()
- source_col[chunk_bounds] = np.nan
+ if len(chunk_bounds) > 0:
+ source_col[chunk_bounds] = np.nan
backprojected = source_col.reindex(target_flagscol.index, method="bfill", tolerance=freq)
fwrdprojected = source_col.reindex(target_flagscol.index, method="ffill", tolerance=freq)
- b_replacement_mask = (backprojected > target_flagscol) & (backprojected >= fwrdprojected)
- f_replacement_mask = (fwrdprojected > target_flagscol) & (fwrdprojected > backprojected)
- target_flagscol.loc[b_replacement_mask] = backprojected.loc[b_replacement_mask]
- target_flagscol.loc[f_replacement_mask] = fwrdprojected.loc[f_replacement_mask]
- return target_flagscol
+ return pd.concat([backprojected, fwrdprojected], axis=1).max(axis=1)
def _inverseAggregation(target_flagscol, source_col=None, freq=None, method=None):
@@ -743,24 +740,27 @@ def reindexFlags(
target_datcol = data[field]
target_flagscol = flagger[field]
- append_dummy = pd.Series(np.nan, target_flagscol.index)
+ blank_dummy = pd.Series(np.nan, target_flagscol.index)
if method[-13:] == "interpolation":
ignore = _getChunkBounds(target_datcol, flagscol, freq)
merge_func = _inverseInterpolation
- merge_dict = dict(freq=freq, chunk_bounds=ignore)
+ merge_dict = dict(freq=freq, chunk_bounds=ignore, target_flagscol=blank_dummy)
+ mask_dict = {**merge_dict, 'chunk_bounds':[]}
if method[-3:] == "agg" or method == "match":
projection_method = METHOD2ARGS[method][0]
tolerance = METHOD2ARGS[method][1](freq)
merge_func = _inverseAggregation
- merge_dict = dict(freq=tolerance, method=projection_method)
+ merge_dict = mask_dict = dict(freq=tolerance, method=projection_method, target_flagscol=blank_dummy)
if method[-5:] == "shift":
drop_mask = (target_datcol.isna() | isflagged(target_flagscol, kwargs['to_mask']))
projection_method = METHOD2ARGS[method][0]
tolerance = METHOD2ARGS[method][1](freq)
merge_func = _inverseShift
- merge_dict = dict(freq=tolerance, method=projection_method, drop_mask=drop_mask)
+ merge_dict = mask_dict = dict(freq=tolerance, method=projection_method, drop_mask=drop_mask, target_flagscol=blank_dummy)
- flagger = mergeHistoryByFunc(flagger, field, source, merge_func, merge_dict, last_column=append_dummy)
+ tmp_flagger = applyFunctionOnHistory(flagger, source, merge_func, merge_dict, merge_func, mask_dict,
+ last_column=blank_dummy)
+ flagger = appendHistory(flagger, field, tmp_flagger.history[source])
return data, flagger
--
GitLab
From 864b471717e695cfd6f12833f3da6779f012ceea Mon Sep 17 00:00:00 2001
From: Peter Luenenschloss <peter.luenenschloss@ufz.de>
Date: Mon, 22 Mar 2021 16:41:10 +0100
Subject: [PATCH 066/180] inverse aggregation running
---
saqc/funcs/resampling.py | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index decac74df..f35d4b17c 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -630,11 +630,9 @@ def _inverseInterpolation(source_col, freq=None, chunk_bounds=None, target_flags
return pd.concat([backprojected, fwrdprojected], axis=1).max(axis=1)
-def _inverseAggregation(target_flagscol, source_col=None, freq=None, method=None):
- source_col = source_col.reindex(target_flagscol.index, method=method, tolerance=freq)
- replacement_mask = source_col > target_flagscol
- target_flagscol.loc[replacement_mask] = source_col.loc[replacement_mask]
- return target_flagscol
+def _inverseAggregation(source_col, freq=None, method=None, target_flagscol=None):
+ return source_col.reindex(target_flagscol.index, method=method, tolerance=freq)
+
def _inverseShift(target_flagscol, source_col=None, freq=None, method=None, drop_mask=None):
--
GitLab
From 1b157b22c756c7acedbe1574d773f79b8ccf90a3 Mon Sep 17 00:00:00 2001
From: Peter Luenenschloss <peter.luenenschloss@ufz.de>
Date: Mon, 22 Mar 2021 17:00:47 +0100
Subject: [PATCH 067/180] shift running
---
saqc/funcs/resampling.py | 16 ++++++----------
1 file changed, 6 insertions(+), 10 deletions(-)
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index f35d4b17c..482fdb02b 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -635,9 +635,9 @@ def _inverseAggregation(source_col, freq=None, method=None, target_flagscol=None
-def _inverseShift(target_flagscol, source_col=None, freq=None, method=None, drop_mask=None):
+def _inverseShift(source_col, freq=None, method=None, drop_mask=None, target_flagscol=None):
target_flagscol_drops = target_flagscol[drop_mask]
- target_flagscol.drop(drop_mask[drop_mask].index, inplace=True)
+ target_flagscol = target_flagscol.drop(drop_mask[drop_mask].index)
flags_merged = pd.merge_asof(
source_col,
pd.Series(target_flagscol.index.values, index=target_flagscol.index, name="pre_index"),
@@ -647,17 +647,13 @@ def _inverseShift(target_flagscol, source_col=None, freq=None, method=None, drop
direction=method,
)
flags_merged.dropna(subset=["pre_index"], inplace=True)
- flags_merged = flags_merged.set_index(["pre_index"]).squeeze()
-
- # write flags to target
- replacement_mask = flags_merged > target_flagscol.loc[flags_merged.index]
- target_flagscol.loc[replacement_mask[replacement_mask].index] = flags_merged.loc[replacement_mask]
+ target_flagscol = flags_merged.set_index(["pre_index"]).squeeze()
# reinsert drops
- target_flagscol = target_flagscol.reindex(target_flagscol.index.join(target_flagscol_drops.index, how="outer"))
- target_flagscol.loc[target_flagscol_drops.index] = target_flagscol_drops.values
+ source_col = target_flagscol.reindex(target_flagscol.index.join(target_flagscol_drops.index, how="outer"))
+ source_col.loc[target_flagscol_drops.index] = target_flagscol_drops.values
- return target_flagscol
+ return source_col
@register(masking='none', module="resampling")
--
GitLab
From bb79227092c56677fdd160910ce3f0c934289005 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Mon, 22 Mar 2021 14:20:37 +0100
Subject: [PATCH 068/180] fixed shift
---
saqc/funcs/resampling.py | 87 ++++++++++++++++------------------------
1 file changed, 35 insertions(+), 52 deletions(-)
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 482fdb02b..4e9dad616 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -361,21 +361,13 @@ def _shift(
field: str,
flagger: Flagger,
freq: str,
- method: Literal["fshift", "bshift", "nshift"]="nshift",
- to_drop: Optional[Union[Any, Sequence[Any]]]=None,
- empty_intervals_flag: float = UNFLAGGED,
- freq_check: Optional[Literal["check", "auto"]]=None,
+ method: Literal["fshift", "bshift", "nshift"] = "nshift",
+ freq_check: Optional[Literal["check", "auto"]] = None,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
Function to shift data points to regular (equidistant) timestamps.
- Values get shifted according to the keyword passed to the `method` parameter.
-
- * ``'nshift'``: every grid point gets assigned the nearest value in its range. (range = +/- 0.5 * `freq`)
- * ``'bshift'``: every grid point gets assigned its first succeeding value - if there is one available in the
- succeeding sampling interval.
- * ``'fshift'``: every grid point gets assigned its ultimately preceeding value - if there is one available in
- the preceeding sampling interval.
+ Values and Flags get shifted according to the keyword passed to the `method` parameter.
Note: all data nans get excluded defaultly from shifting. If `to_drop` is ``None``, - all *BAD* flagged values get
excluded as well.
@@ -384,27 +376,32 @@ def _shift(
----------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
+
field : str
The fieldname of the column, holding the data-to-be-shifted.
+
flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
+
freq : str
An frequency Offset String that will be interpreted as the sampling rate you want the data to be shifted to.
- method: {'fshift', 'bshift', 'nshift'}, default 'nshift'
- Specifies if datapoints get propagated forwards, backwards or to the nearest grid timestamp. See function
- description for more details.
- empty_intervals_flag : float, default UNFLAGGED
- The Flag, that is assigned to grid points, if no values are available to be shifted to.
- to_drop : {None, str, List[str]}, default None
- Flags that refer to values you want to drop before shifting - effectively, excluding values that are flagged
- with a flag in to_drop from the shifting process. Default - to_drop = None - results in BAD
- values being dropped initially.
+
+ method : {'fshift', 'bshift', 'nshift'}, default 'nshift'
+ Specifies how misaligned data-points get propagated to a grid timestamp.
+ Following choices are available:
+
+ * 'nshift' : every grid point gets assigned the nearest value in its range. (range = +/- 0.5 * `freq`)
+ * 'bshift' : every grid point gets assigned its first succeeding value, if one is available in
+ the succeeding sampling interval.
+ * 'fshift' : every grid point gets assigned its ultimately preceding value, if one is available in
+ the preceeding sampling interval.
+
freq_check : {None, 'check', 'auto'}, default None
- * ``None``: do not validate frequency-string passed to `freq`
- * ``'check'``: estimate frequency and log a warning if estimate miss matches frequency string passed to `freq`,
+ * ``None`` : do not validate frequency-string passed to `freq`
+ * 'check' : estimate frequency and log a warning if estimate miss matches frequency string passed to `freq`,
or if no uniform sampling rate could be estimated
- * ``'auto'``: estimate frequency and use estimate. (Ignores `freq` parameter.)
+ * 'auto' : estimate frequency and use estimate. (Ignores `freq` parameter.)
Returns
-------
@@ -415,40 +412,26 @@ def _shift(
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
- data = data.copy()
+ flagged = isflagged(flagger[field], kwargs['to_mask'])
datcol = data[field]
- flagscol = flagger[field]
-
- drop_mask = getDropMask(field, to_drop, flagger, BAD)
- drop_mask |= datcol.isna()
- datcol[drop_mask] = np.nan
- datcol.dropna(inplace=True)
- flagscol.drop(drop_mask[drop_mask].index, inplace=True)
-
- # create a dummys
- if datcol.empty:
- datcol = pd.Series([], index=pd.DatetimeIndex([]), name=field)
- flagscol = pd.Series([], index=pd.DatetimeIndex([]), name=field)
-
- # clear the past
- flagger.history[field] = flagger.history[field].reindex(datcol.index)
- flagger[field] = flagscol
+ datcol[flagged] = np.nan
+ freq = evalFreqStr(freq, freq_check, datcol.index)
- # do the shift, we need to process the history manually
- else:
- freq = evalFreqStr(freq, freq_check, datcol.index)
- datcol = shift2Freq(datcol, method, freq, fill_value=np.nan)
+ # do the shift
+ datcol = shift2Freq(datcol, method, freq, fill_value=np.nan)
- # after next 3 lines we leave history in unstable state
- # but the following append will fix this
- history = flagger.history[field]
- history.hist = shift2Freq(history.hist, method, freq, fill_value=UNTOUCHED)
- history.mask = shift2Freq(history.mask, method, freq, fill_value=False)
+ # do the shift on the history
+ history = flagger.history[field]
+ history.hist = shift2Freq(history.hist, method, freq, fill_value=UNTOUCHED)
+ history.mask = shift2Freq(history.mask, method, freq, fill_value=False)
- flagscol = shift2Freq(flagscol, method, freq, fill_value=empty_intervals_flag)
- history.append(flagscol, force=True)
- flagger.history[field] = history
+ # The last 2 lines left the history in an unstable state, Also we want to
+ # append a dummy column, that represent the 'shift' in the history.
+ # Luckily the append also fix the unstable state - noice.
+ dummy = pd.Series(UNTOUCHED, index=datcol.index, dtype=float)
+ history.append(dummy, force=True)
+ flagger.history[field] = history
data[field] = datcol
return data, flagger
--
GitLab
From c9b6fe73c5bebf54e5a7281aa539612c884b55f2 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Mon, 22 Mar 2021 16:38:06 +0100
Subject: [PATCH 069/180] fixed shift and resamle
---
saqc/funcs/resampling.py | 157 ++++++++++++++++-----------------------
1 file changed, 64 insertions(+), 93 deletions(-)
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 4e9dad616..e304cc54f 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -337,40 +337,16 @@ def mapToOriginal(
@register(masking='none', module="resampling")
def shift(
- data: DictOfSeries,
- field: str,
- flagger: Flagger,
- freq: str,
- method: Literal["fshift", "bshift", "nshift"]="nshift",
- to_drop: Optional[Union[Any, Sequence[Any]]]=None,
- empty_intervals_flag: Optional[str]=None,
- freq_check: Optional[Literal["check", "auto"]]=None, # TODO: not a user decision
- **kwargs
-) -> Tuple[DictOfSeries, Flagger]:
-
- data, flagger = copy(data, field, flagger, field + '_original')
- data, flagger = _shift(
- data, field, flagger, freq, method=method, to_drop=to_drop,
- empty_intervals_flag=empty_intervals_flag, freq_check=freq_check, **kwargs
- )
- return data, flagger
-
-
-def _shift(
data: DictOfSeries,
field: str,
flagger: Flagger,
freq: str,
method: Literal["fshift", "bshift", "nshift"] = "nshift",
- freq_check: Optional[Literal["check", "auto"]] = None,
+ freq_check: Optional[Literal["check", "auto"]] = None, # TODO: not a user decision
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
- Function to shift data points to regular (equidistant) timestamps.
- Values and Flags get shifted according to the keyword passed to the `method` parameter.
-
- Note: all data nans get excluded defaultly from shifting. If `to_drop` is ``None``, - all *BAD* flagged values get
- excluded as well.
+ Function to shift data and flags to a regular (equidistant) timestamp grid, according to ``method``.
Parameters
----------
@@ -412,6 +388,26 @@ def _shift(
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
+ data, flagger = copy(data, field, flagger, field + '_original')
+ return _shift(data, field, flagger, freq, method=method, freq_check=freq_check, **kwargs)
+
+
+def _shift(
+ data: DictOfSeries,
+ field: str,
+ flagger: Flagger,
+ freq: str,
+ method: Literal["fshift", "bshift", "nshift"] = "nshift",
+ freq_check: Optional[Literal["check", "auto"]] = None,
+ **kwargs
+) -> Tuple[DictOfSeries, Flagger]:
+ """
+ Function to shift data points to regular (equidistant) timestamps.
+
+ See Also
+ --------
+ shift : Main caller, docstring
+ """
flagged = isflagged(flagger[field], kwargs['to_mask'])
datcol = data[field]
datcol[flagged] = np.nan
@@ -436,7 +432,7 @@ def _shift(
return data, flagger
-@register(masking='field', module="resampling")
+@register(masking='none', module="resampling")
def resample(
data: DictOfSeries,
field: str,
@@ -449,9 +445,6 @@ def resample(
max_invalid_consec_f: Optional[int]=None,
max_invalid_total_f: Optional[int]=None,
flag_agg_func: Callable[[pd.Series], float]=max,
- empty_intervals_flag: float = BAD,
- to_drop: Optional[Union[Any, Sequence[Any]]]=None,
- all_na_2_empty: bool=False,
freq_check: Optional[Literal["check", "auto"]]=None,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
@@ -480,45 +473,48 @@ def resample(
----------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
+
field : str
The fieldname of the column, holding the data-to-be-resampled.
+
flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
+
freq : str
An Offset String, that will be interpreted as the frequency you want to resample your data with.
+
agg_func : Callable
The function you want to use for aggregation.
+
method: {'fagg', 'bagg', 'nagg'}, default 'bagg'
Specifies which intervals to be aggregated for a certain timestamp. (preceding, succeeding or
"surrounding" interval). See description above for more details.
+
max_invalid_total_d : {None, int}, default None
Maximum number of invalid (nan) datapoints, allowed per resampling interval. If max_invalid_total_d is
exceeded, the interval gets resampled to nan. By default (``np.inf``), there is no bound to the number of nan
values in an interval and only intervals containing ONLY nan values or those, containing no values at all,
get projected onto nan
+
max_invalid_consec_d : {None, int}, default None
Maximum number of consecutive invalid (nan) data points, allowed per resampling interval.
If max_invalid_consec_d is exceeded, the interval gets resampled to nan. By default (np.inf),
there is no bound to the number of consecutive nan values in an interval and only intervals
containing ONLY nan values, or those containing no values at all, get projected onto nan.
+
max_invalid_total_f : {None, int}, default None
Same as `max_invalid_total_d`, only applying for the flags. The flag regarded as "invalid" value,
is the one passed to empty_intervals_flag (default=``BAD``).
Also this is the flag assigned to invalid/empty intervals.
+
max_invalid_consec_f : {None, int}, default None
Same as `max_invalid_total_f`, only applying onto flags. The flag regarded as "invalid" value, is the one passed
to empty_intervals_flag. Also this is the flag assigned to invalid/empty intervals.
+
flag_agg_func : Callable, default: max
The function you want to aggregate the flags with. It should be capable of operating on the flags dtype
(usually ordered categorical).
- empty_intervals_flag : float, default BAD
- A Flag, that you want to assign to invalid intervals. Invalid are those intervals, that contain nan values only,
- or no values at all. Furthermore the empty_intervals_flag is the flag, serving as "invalid" identifyer when
- checking for `max_total_invalid_f` and `max_consec_invalid_f patterns`.
- to_drop : {None, str, List[str]}, default None
- Flags that refer to values you want to drop before resampling - effectively excluding values that are flagged
- with a flag in to_drop from the resampling process - this means that they also will not be counted in the
- the `max_consec`/`max_total evaluation`. `to_drop` = ``None`` results in NO flags being dropped initially.
+
freq_check : {None, 'check', 'auto'}, default None
* ``None``: do not validate frequency-string passed to `freq`
@@ -535,63 +531,38 @@ def resample(
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
-
- data = data.copy()
+ flagged = isflagged(flagger[field], kwargs['to_mask'])
datcol = data[field]
- flagscol = flagger[field]
-
- drop_mask = getDropMask(field, to_drop, flagger, [])
- datcol.drop(datcol[drop_mask].index, inplace=True)
+ datcol[flagged] = np.nan
freq = evalFreqStr(freq, freq_check, datcol.index)
- flagscol.drop(flagscol[drop_mask].index, inplace=True)
-
- # create a dummys
- if all_na_2_empty and datcol.dropna().empty:
- # Todo: This needs discussion. See issue #GL170
- datcol = pd.Series([], index=pd.DatetimeIndex([]), name=field)
- flagscol = pd.Series([], index=pd.DatetimeIndex([]), name=field)
-
- # clear the past
- flagger.history[field] = flagger.history[field].reindex(datcol.index)
- flagger[field] = flagscol
-
- # do the resampling
- else:
- datcol = aggregate2Freq(
- datcol,
- method,
- freq,
- agg_func,
- fill_value=np.nan,
- max_invalid_total=max_invalid_total_d,
- max_invalid_consec=max_invalid_consec_d,
- )
-
- flagscol = aggregate2Freq(
- flagscol,
- method,
- freq,
- flag_agg_func,
- fill_value=empty_intervals_flag,
- max_invalid_total=max_invalid_total_f,
- max_invalid_consec=max_invalid_consec_f,
- )
-
- kws = dict(
- method=method,
- freq=freq,
- agg_func=flag_agg_func,
- fill_value=UNTOUCHED,
- max_invalid_total=max_invalid_total_f,
- max_invalid_consec=max_invalid_consec_f,
- )
-
- flagger = applyFunctionOnHistory(
- flagger, field,
- hist_func=aggregate2Freq, hist_kws=kws,
- mask_func=aggregate2Freq, mask_kws=kws,
- last_column=flagscol
- )
+
+ datcol = aggregate2Freq(
+ datcol,
+ method,
+ freq,
+ agg_func,
+ fill_value=np.nan,
+ max_invalid_total=max_invalid_total_d,
+ max_invalid_consec=max_invalid_consec_d,
+ )
+
+ dummy = pd.Series(UNTOUCHED, index=datcol.index, dtype=float)
+
+ kws = dict(
+ method=method,
+ freq=freq,
+ agg_func=flag_agg_func,
+ fill_value=UNTOUCHED,
+ max_invalid_total=max_invalid_total_f,
+ max_invalid_consec=max_invalid_consec_f,
+ )
+
+ flagger = applyFunctionOnHistory(
+ flagger, field,
+ hist_func=aggregate2Freq, hist_kws=kws,
+ mask_func=aggregate2Freq, mask_kws=kws,
+ last_column=dummy
+ )
data[field] = datcol
return data, flagger
--
GitLab
From b6349c99e3f767fbc71b7fff3fd0eb8dd4fa773e Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Mon, 22 Mar 2021 16:38:48 +0100
Subject: [PATCH 070/180] imporved harm tests
---
tests/common.py | 46 ++++++++++++++
tests/funcs/test_harm_funcs.py | 112 ++++++++++++++++++++++-----------
2 files changed, 121 insertions(+), 37 deletions(-)
diff --git a/tests/common.py b/tests/common.py
index 3e70ff349..e225b6410 100644
--- a/tests/common.py
+++ b/tests/common.py
@@ -42,3 +42,49 @@ def writeIO(content):
return f
+def checkDataFlaggerInvariants(data, flagger, field, identical=True):
+ """
+ Check all invariants that must hold at any point for
+ * field
+ * data
+ * flagger
+ * data[field]
+ * flagger[field]
+ * data[field].index
+ * flagger[field].index
+ * between data and flagger
+ * between data[field] and flagger[field]
+
+ Parameters
+ ----------
+ data : dios.DictOfSeries
+ data container
+ flagger : Flags
+ flags container
+ field : str
+ the field in question
+ identical : bool, default True
+ whether to check indexes of data and flagger to be
+ identical (True, default) of just for equality.
+ """
+ assert isinstance(data, dios.DictOfSeries)
+ assert isinstance(flagger, Flagger)
+
+ # all columns in data are in flagger
+ assert data.columns.difference(flagger.columns).empty
+
+ # ------------------------------------------------------------------------
+ # below here, we just check on and with field
+ # ------------------------------------------------------------------------
+ assert field in data
+ assert field in flagger
+
+ assert flagger[field].dtype == float
+
+ # `pd.Index.identical` also check index attributes like `freq`
+ if identical:
+ assert data[field].index.identical(flagger[field].index)
+ else:
+ assert data[field].index.equals(flagger[field].index)
+
+
diff --git a/tests/funcs/test_harm_funcs.py b/tests/funcs/test_harm_funcs.py
index 3b1ec42c6..7cc99a7cf 100644
--- a/tests/funcs/test_harm_funcs.py
+++ b/tests/funcs/test_harm_funcs.py
@@ -16,6 +16,8 @@ from saqc.funcs.resampling import (
mapToOriginal,
)
+from tests.common import checkDataFlaggerInvariants
+
@pytest.fixture
def data():
@@ -33,6 +35,74 @@ def data():
return data
+@pytest.mark.parametrize('func, kws', [
+ ('linear', dict()),
+ ('shift', dict(method="nshift")),
+ ('interpolate', dict(method="spline")),
+ ('aggregate', dict(value_func=np.nansum, method="nagg")),
+])
+def test_wrapper(data, func, kws):
+ field = 'data'
+ freq = "15min"
+ flagger = initFlagsLike(data)
+
+ import saqc
+ func = getattr(saqc.funcs, func)
+ data, flagger = func(data, field, flagger, freq, **kws)
+
+ # check minimal requirements
+ checkDataFlaggerInvariants(data, flagger, field)
+ assert data[field].index.freq == pd.Timedelta(freq)
+
+
+@pytest.mark.parametrize("method", ["time", "polynomial"])
+def test_gridInterpolation(data, method):
+ freq = "15min"
+ field = 'data'
+ data = data[field]
+ data = (data * np.sin(data)).append(data.shift(1, "2h")).shift(1, "3s")
+ data = dios.DictOfSeries(data)
+ flagger = initFlagsLike(data)
+
+ # we are just testing if the interpolation gets passed to the series without causing an error:
+ res = interpolate(data, field, flagger, freq, method=method, downcast_interpolation=True)
+
+ if method == "polynomial":
+ res = interpolate(data, field, flagger, freq, order=2, method=method, downcast_interpolation=True)
+ res = interpolate(data, field, flagger, freq, order=10, method=method, downcast_interpolation=True)
+
+ # check minimal requirements
+ rdata, rflagger = res
+ checkDataFlaggerInvariants(rdata, rflagger, field, identical=False)
+ assert rdata[field].index.freq == pd.Timedelta(freq)
+
+
+@pytest.mark.parametrize('func, kws', [
+ ('linear', dict()),
+ ('shift', dict(method="nshift")),
+ ('interpolate', dict(method="spline")),
+ ('aggregate', dict(value_func=np.nansum, method="nagg")),
+])
+def test_flagsSurviveReshaping(reshaper):
+ """
+ flagging -> reshaping -> test (flags also was reshaped correctly)
+ """
+ pass
+
+
+def test_flagsSurviveInverseReshaping():
+ """
+ inverse reshaping -> flagging -> test (flags also was reshaped correctly)"""
+ pass
+
+
+def test_flagsSurviveBackprojection():
+ """
+ flagging -> reshaping -> inverse reshaping -> test (flags == original-flags)
+ """
+ pass
+
+
@pytest.mark.parametrize("reshaper", ["nshift", "fshift", "bshift", "nagg", "bagg", "fagg", "interpolation"])
def test_harmSingleVarIntermediateFlagging(data, reshaper):
flagger = initFlagsLike(data)
@@ -96,6 +166,7 @@ def test_harmSingleVarIntermediateFlagging(data, reshaper):
def test_harmSingleVarInterpolationAgg(data, params, expected):
flagger = initFlagsLike(data)
field = 'data'
+
pre_data = data.copy()
pre_flaggger = flagger.copy()
method, freq = params
@@ -111,14 +182,14 @@ def test_harmSingleVarInterpolationAgg(data, params, expected):
@pytest.mark.parametrize(
'params, expected',
[
- (("fshift", "15Min"), pd.Series(data=[np.nan, -37.5, -25.0, 0.0, 37.5, 50.0], index=pd.date_range("2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15Min"))),
- (("fshift", "30Min"), pd.Series(data=[np.nan, -37.5, 0.0, 50.0], index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"))),
(("bshift", "15Min"), pd.Series(data=[-50.0, -37.5, -25.0, 12.5, 37.5, 50.0], index=pd.date_range("2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15Min"))),
- (("bshift", "30Min"), pd.Series(data=[-50.0, -37.5, 12.5, 50.0], index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"))),
+ (("fshift", "15Min"), pd.Series(data=[np.nan, -37.5, -25.0, 0.0, 37.5, 50.0], index=pd.date_range("2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15Min"))),
(("nshift", "15min"), pd.Series(data=[np.nan, -37.5, -25.0, 12.5, 37.5, 50.0], index=pd.date_range("2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15Min"))),
+ (("bshift", "30Min"), pd.Series(data=[-50.0, -37.5, 12.5, 50.0], index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"))),
+ (("fshift", "30Min"), pd.Series(data=[np.nan, -37.5, 0.0, 50.0], index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"))),
(("nshift", "30min"), pd.Series(data=[np.nan, -37.5, 12.5, 50.0], index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"))),
])
-def test_harmSingleVarInterpolationShift(data, params, expected):
+def test_harmSingleVarInterpolationShift(data, params, expected):
flagger = initFlagsLike(data)
field = 'data'
pre_data = data.copy()
@@ -133,36 +204,3 @@ def test_harmSingleVarInterpolationShift(data, params, expected):
assert flagger_deharm[field].equals(pre_flagger[field])
-@pytest.mark.parametrize("method", ["time", "polynomial"])
-def test_gridInterpolation(data, method):
- freq = "15min"
- field = 'data'
- data = data[field]
- data = (data * np.sin(data)).append(data.shift(1, "2h")).shift(1, "3s")
- data = dios.DictOfSeries(data)
- flagger = initFlagsLike(data)
-
- # we are just testing if the interpolation gets passed to the series without causing an error:
- interpolate(data, field, flagger, freq, method=method, downcast_interpolation=True)
-
- if method == "polynomial":
- interpolate(data, field, flagger, freq, order=2, method=method, downcast_interpolation=True)
- interpolate(data, field, flagger, freq, order=10, method=method, downcast_interpolation=True)
-
-
-@pytest.mark.parametrize('func, kws', [
- ('linear', dict(to_drop=None)),
- ('shift', dict(method="nshift", to_drop=None)),
- ('interpolate', dict(method="spline")),
- ('aggregate', dict(value_func=np.nansum, method="nagg", to_drop=None)),
-])
-def test_wrapper(data, func, kws):
- # we are only testing, whether the wrappers do pass processing:
- field = 'data'
- freq = "15min"
- flagger = initFlagsLike(data)
-
- import saqc
- func = getattr(saqc.funcs, func)
- func(data, field, flagger, freq, **kws)
-
--
GitLab
From ade732d06716d7f060ae655196a87bc0a9f3c0b3 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Mon, 22 Mar 2021 17:17:08 +0100
Subject: [PATCH 071/180] tests
---
tests/funcs/test_harm_funcs.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/tests/funcs/test_harm_funcs.py b/tests/funcs/test_harm_funcs.py
index 7cc99a7cf..e09177a01 100644
--- a/tests/funcs/test_harm_funcs.py
+++ b/tests/funcs/test_harm_funcs.py
@@ -83,7 +83,7 @@ def test_gridInterpolation(data, method):
('interpolate', dict(method="spline")),
('aggregate', dict(value_func=np.nansum, method="nagg")),
])
-def test_flagsSurviveReshaping(reshaper):
+def test_flagsSurviveReshaping(func, kws):
"""
flagging -> reshaping -> test (flags also was reshaped correctly)
"""
@@ -112,6 +112,8 @@ def test_harmSingleVarIntermediateFlagging(data, reshaper):
pre_flagger = flagger.copy()
data, flagger = linear(data, field, flagger, freq="15min")
+ checkDataFlaggerInvariants(data, flagger, field, identical=True)
+ assert data[field].index.freq == pd.Timedelta('15min')
# flag something bad
flagger[data[field].index[3:4], field] = BAD
@@ -172,9 +174,12 @@ def test_harmSingleVarInterpolationAgg(data, params, expected):
method, freq = params
data_harm, flagger_harm = aggregate(data, field, flagger, freq, value_func=np.sum, method=method)
+ checkDataFlaggerInvariants(data_harm, flagger_harm, field, identical=True)
+ assert data_harm[field].index.freq == pd.Timedelta(freq)
assert data_harm[field].equals(expected)
data_deharm, flagger_deharm = mapToOriginal(data_harm, "data", flagger_harm, method="inverse_" + method)
+ checkDataFlaggerInvariants(data_harm, flagger_harm, field, identical=True)
assert data_deharm[field].equals(pre_data[field])
assert flagger_deharm[field].equals(pre_flaggger[field])
--
GitLab
From 4c5053356e048ac4db88fbe3c55b6dfcf3e0853b Mon Sep 17 00:00:00 2001
From: Peter Luenenschloss <peter.luenenschloss@ufz.de>
Date: Mon, 22 Mar 2021 17:49:10 +0100
Subject: [PATCH 072/180] shift tests running now (tiny bug in _inverse shift)
---
saqc/funcs/resampling.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index e304cc54f..40e2bd50f 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -601,7 +601,8 @@ def _inverseShift(source_col, freq=None, method=None, drop_mask=None, target_fla
direction=method,
)
flags_merged.dropna(subset=["pre_index"], inplace=True)
- target_flagscol = flags_merged.set_index(["pre_index"]).squeeze()
+ flags_merged = flags_merged.set_index(["pre_index"]).squeeze()
+ target_flagscol[flags_merged.index] = flags_merged.values
# reinsert drops
source_col = target_flagscol.reindex(target_flagscol.index.join(target_flagscol_drops.index, how="outer"))
--
GitLab
From 74f8c66babb4758f64777ff7a049399192c62d76 Mon Sep 17 00:00:00 2001
From: Peter Luenenschloss <peter.luenenschloss@ufz.de>
Date: Mon, 22 Mar 2021 17:57:34 +0100
Subject: [PATCH 073/180] removed peculiar neg-operator -> harm tests running
again
---
tests/funcs/test_harm_funcs.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tests/funcs/test_harm_funcs.py b/tests/funcs/test_harm_funcs.py
index e09177a01..052d5f8da 100644
--- a/tests/funcs/test_harm_funcs.py
+++ b/tests/funcs/test_harm_funcs.py
@@ -134,8 +134,8 @@ def test_harmSingleVarIntermediateFlagging(data, reshaper):
raise NotImplementedError('untested test case')
assert all(flagger[field].iloc[start:end] > UNFLAGGED)
- assert all(~flagger[field].iloc[:start] == UNFLAGGED)
- assert all(~flagger[field].iloc[end:] == UNFLAGGED)
+ assert all(flagger[field].iloc[:start] == UNFLAGGED)
+ assert all(flagger[field].iloc[end:] == UNFLAGGED)
elif 'shift' in reshaper:
if reshaper == "nshift":
--
GitLab
From 86e6c765a1c8bd062e3a8ebd0bc68f53759936bc Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 23 Mar 2021 01:15:00 +0100
Subject: [PATCH 074/180] refactors and fixed test_modelling.py
---
saqc/core/register.py | 3 +-
saqc/flagger/flags.py | 30 +++++++-
saqc/flagger/history.py | 11 ++-
saqc/funcs/interpolation.py | 141 +++++++++++++++++-----------------
saqc/funcs/resampling.py | 78 +++++++++++--------
saqc/lib/ts_operators.py | 20 +++--
tests/funcs/test_modelling.py | 41 ++++++----
7 files changed, 191 insertions(+), 133 deletions(-)
diff --git a/saqc/core/register.py b/saqc/core/register.py
index ce88dc4bc..39c1d4b14 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -49,7 +49,8 @@ def register(masking: MaskingStrT = "all", module: Optional[str] = None):
# executed if a register-decorated function is called,
# nevertheless if it is called plain or via `SaQC.func`.
@wraps(func)
- def callWrapper(*args, **kwargs):
+ def callWrapper(data, field, flagger, *args, **kwargs):
+ args = data, field, flagger, *args
args, kwargs, old_state = _preCall(func, args, kwargs, masking, func_name)
result = func(*args, **kwargs)
return _postCall(result, old_state)
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index c1dcb1ed6..28a0cef59 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -36,6 +36,12 @@ class _HistAccess:
def __setitem__(self, key: str, value: Union[History, pd.DataFrame]):
if not isinstance(value, History):
value = History(value)
+
+ if not isinstance(value, History):
+ raise TypeError("Not a History")
+
+ History._validate_hist_with_mask(value.hist, value.mask)
+
self.obj._data[key] = value
self.obj._cache.pop(key, None)
@@ -339,7 +345,9 @@ def initFlagsLike(
return Flags(result)
-def applyFunctionOnHistory(flags: Flags, column, hist_func, hist_kws, mask_func, mask_kws, last_column=None):
+def applyFunctionOnHistory(
+ flags: Flags, column, hist_func, hist_kws, mask_func, mask_kws, last_column=None, func_handle_df=False
+):
"""
Apply function on history.
@@ -355,6 +363,7 @@ def applyFunctionOnHistory(flags: Flags, column, hist_func, hist_kws, mask_func,
mask_func :
mask_kws :
last_column :
+ func_handle_df :
Returns
-------
@@ -363,15 +372,28 @@ def applyFunctionOnHistory(flags: Flags, column, hist_func, hist_kws, mask_func,
flags = flags.copy()
history = flags.history[column]
new_history = History()
- for pos in history.columns:
- new_history.hist[pos] = hist_func(history.hist[pos], **hist_kws)
- new_history.mask[pos] = mask_func(history.mask[pos], **mask_kws)
+ if func_handle_df:
+ history.hist = hist_func(history.hist, **hist_kws)
+ history.mask = hist_func(history.mask, **mask_kws)
+
+ else:
+ for pos in history.columns:
+ new_history.hist[pos] = hist_func(history.hist[pos], **hist_kws)
+ new_history.mask[pos] = mask_func(history.mask[pos], **mask_kws)
+
+ # handle unstable state
if last_column is None:
new_history.mask.iloc[:, -1:] = True
else:
+ if isinstance(last_column, str) and last_column == 'dummy':
+ last_column = pd.Series(UNTOUCHED, index=new_history.mask.index, dtype=float)
+
new_history.append(last_column, force=True)
+ # assure a boolean mask
+ new_history.mask = new_history.mask.fillna(False).astype(bool)
+
flags.history[column] = new_history
return flags
diff --git a/saqc/flagger/history.py b/saqc/flagger/history.py
index 72a573bd1..2acc8f22e 100644
--- a/saqc/flagger/history.py
+++ b/saqc/flagger/history.py
@@ -333,13 +333,14 @@ class History:
# validation
#
- def _validate_hist_with_mask(self, obj: pd.DataFrame, mask: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]:
+ @staticmethod
+ def _validate_hist_with_mask(obj: pd.DataFrame, mask: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]:
"""
check type, columns, index, dtype and if the mask fits the obj.
"""
# check hist
- self._validate_hist(obj)
+ History._validate_hist(obj)
# check mask
if not isinstance(mask, pd.DataFrame):
@@ -360,7 +361,8 @@ class History:
return obj, mask
- def _validate_hist(self, obj: pd.DataFrame) -> pd.DataFrame:
+ @staticmethod
+ def _validate_hist(obj: pd.DataFrame) -> pd.DataFrame:
"""
check type, columns, dtype of obj.
"""
@@ -379,7 +381,8 @@ class History:
return obj
- def _validate_value(self, obj: pd.Series) -> pd.Series:
+ @staticmethod
+ def _validate_value(obj: pd.Series) -> pd.Series:
"""
index is not checked !
"""
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index c0b9b8ee0..be93d5492 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -34,31 +34,35 @@ def interpolateByRolling(
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
- Interpolates missing values (nan values present in the data) by assigning them the aggregation result of
- a window surrounding them.
-
- Note, that in the current implementation, center=True can only be used with integer window sizes - furthermore
- note, that integer window sizes can yield screwed aggregation results for not-harmonized or irregular data.
+ Interpolates nan-values in the data by assigning them the aggregation result of the window surrounding them.
Parameters
----------
data : dios.DictOfSeries
- A dictionary of pandas.Series, holding all the data.
+ The data container.
+
field : str
- The fieldname of the column, holding the data-to-be-interpolated.
+ Name of the column, holding the data-to-be-interpolated.
+
flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ A flagger object, holding flags and additional Information related to `data`.
+
winsz : int, str
- The size of the window, the aggregation is computed from. Either counted in periods number (Integer passed),
- or defined by a total temporal extension (offset String passed).
+ The size of the window, the aggregation is computed from. An integer define the number of periods to be used,
+ an string is interpreted as an offset. ( see `pandas.rolling` for more information).
+ Integer windows may result in screwed aggregations if called on none-harmonized or irregular data.
+
func : Callable
The function used for aggregation.
+
center : bool, default True
- Wheather or not the window, the aggregation is computed of, is centered around the value to be interpolated.
+ Center the window around the value. Can only be used with integer windows, otherwise it is silently ignored.
+
min_periods : int
Minimum number of valid (not np.nan) values that have to be available in a window for its aggregation to be
computed.
- flag : float, default UNFLAGGED
+
+ flag : float or None, default UNFLAGGED
Flag that is to be inserted for the interpolated values. If ``None`` no flags are set.
Returns
@@ -83,7 +87,7 @@ def interpolateByRolling(
rolled = roller.apply(func)
na_mask = datcol.isna()
- interpolated = na_mask & ~rolled.isna()
+ interpolated = na_mask & rolled.notna()
datcol[na_mask] = rolled[na_mask]
data[field] = datcol
@@ -102,7 +106,6 @@ def interpolateInvalid(
inter_order: int = 2,
inter_limit: int = 2,
downgrade_interpolation: bool = False,
- not_interpol_flags=None,
flag: float = UNFLAGGED,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
@@ -112,32 +115,36 @@ def interpolateInvalid(
There are available all the interpolation methods from the pandas.interpolate method and they are applicable by
the very same key words, that you would pass to the ``pd.Series.interpolate``'s method parameter.
- Note, that the `inter_limit` keyword really restricts the interpolation to chunks, not containing more than
- `inter_limit` successive nan entries.
-
Parameters
----------
data : dios.DictOfSeries
- A dictionary of pandas.Series, holding all the data.
+ The data container.
+
field : str
- The fieldname of the column, holding the data-to-be-interpolated.
+ Name of the column, holding the data-to-be-interpolated.
+
flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ A flagger object, holding flags and additional Information related to `data`.
+
method : {"linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric",
- "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"}: string
- The interpolation method you want to apply.
+ "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"}
+ The interpolation method to use.
+
inter_order : int, default 2
If there your selected interpolation method can be performed at different 'orders' - here you pass the desired
order.
+
inter_limit : int, default 2
- Maximum number of consecutive 'nan' values allowed for a gap to be interpolated.
+ Maximum number of consecutive 'nan' values allowed for a gap to be interpolated. This really restricts the
+ interpolation to chunks, containing not more than `inter_limit` successive nan entries.
+
flag : float or None, default UNFLAGGED
- Flag that is to be inserted for the interpolated values. If ``None`` no flags are set.
+ Flag that is set for interpolated values. If ``None``, no flags are set at all.
+
downgrade_interpolation : bool, default False
- If interpolation can not be performed at `inter_order`, because not enough values are present or the order
- is not implemented for the passed method, automatically try to interpolate at ``inter_order-1``.
- not_interpol_flags : None
- deprecated
+ If `True` and the interpolation can not be performed at current order, retry with a lower order.
+ This can happen, because the chosen ``method`` does not support the passed ``inter_order``, or
+ simply because not enough values are present in a interval.
Returns
-------
@@ -148,8 +155,6 @@ def interpolateInvalid(
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
"""
-
- data = data.copy()
inter_data = interpolateNANs(
data[field],
method,
@@ -159,10 +164,6 @@ def interpolateInvalid(
)
interpolated = data[field].isna() & inter_data.notna()
- # TODO: remove with version 2.0
- if not_interpol_flags is not None:
- raise ValueError("'not_interpol_flags' is deprecated")
-
if flag is not None:
flagger[interpolated, field] = flag
@@ -170,17 +171,14 @@ def interpolateInvalid(
return data, flagger
-def _overlap_rs(x, freq='1min', fill_value=UNFLAGGED):
- end = x.index[-1].ceil(freq)
- x = x.resample(freq).max()
- x = x.combine(x.shift(1, fill_value=fill_value), max)
- # we are appending last regular grid entry (if necessary), to conserve integrity of groups of regularized
- # timestamps originating all from the same logger.
- try:
- x = x.append(pd.Series([fill_value], index=[end]), verify_integrity=True)
- except ValueError:
- pass
- return x
+def _resampleOverlapping(data: pd.Series, freq: str, fill_value):
+ dtype = data.dtype
+ end = data.index[-1].ceil(freq)
+ data = data.resample(freq).max()
+ data = data.combine(data.shift(1, fill_value=fill_value), max)
+ if end not in data:
+ data.loc[end] = fill_value
+ return data.fillna(fill_value).astype(dtype)
@register(masking='none', module="interpolation")
@@ -191,8 +189,8 @@ def interpolateIndex(
freq: str,
method: _SUPPORTED_METHODS,
inter_order: int = 2,
- downgrade_interpolation: bool = False,
inter_limit: int = 2,
+ downgrade_interpolation: bool = False,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -201,40 +199,38 @@ def interpolateIndex(
Note, that the interpolation will only be calculated, for grid timestamps that have a preceding AND a succeeding
valid data value within "freq" range.
- Note, that the function differs from proc_interpolateMissing, by returning a whole new data set, only containing
- samples at the interpolated, equidistant timestamps (of frequency "freq").
-
- Note, it is possible to interpolate unregular "grids" (with no frequencies). In fact, any date index
- can be target of the interpolation. Just pass the field name of the variable, holding the index
- you want to interpolate, to "grid_field". 'freq' is then use to determine the maximum gap size for
- a grid point to be interpolated.
-
- Note, that intervals, not having an interpolation value assigned (thus, evaluate to np.nan), get UNFLAGGED assigned.
-
Parameters
----------
data : dios.DictOfSeries
- A dictionary of pandas.Series, holding all the data.
+ The data container.
+
field : str
- The fieldname of the column, holding the data-to-be-interpolated.
+ Name of the column, holding the data-to-be-interpolated.
+
flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ A flagger object, holding flags and additional Information related to `data`.
+
freq : str
An Offset String, interpreted as the frequency of
the grid you want to interpolate your data at.
+
method : {"linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric",
"polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"}: string
The interpolation method you want to apply.
- inter_order : integer, default 2
+
+ inter_order : int, default 2
If there your selected interpolation method can be performed at different 'orders' - here you pass the desired
order.
+
+ inter_limit : int, default 2
+ Maximum number of consecutive 'nan' values allowed for a gap to be interpolated. This really restricts the
+ interpolation to chunks, containing not more than `inter_limit` successive nan entries.
+
downgrade_interpolation : bool, default False
- If interpolation can not be performed at `inter_order` - (not enough values or not implemented at this order) -
- automatically try to interpolate at order `inter_order` :math:`- 1`.
- inter_limit : Integer, default 2
- Maximum number of consecutive Grid values allowed for interpolation. If set
- to *n*, chunks of *n* and more consecutive grid values, where there is no value in between, wont be
- interpolated.
+ If `True` and the interpolation can not be performed at current order, retry with a lower order.
+ This can happen, because the chosen ``method`` does not support the passed ``inter_order``, or
+ simply because not enough values are present in a interval.
+
Returns
-------
@@ -254,7 +250,7 @@ def interpolateIndex(
start, end = datcol.index[0].floor(freq), datcol.index[-1].ceil(freq)
grid_index = pd.date_range(start=start, end=end, freq=freq, name=datcol.index.name)
- flagged = isflagged(flagscol, kwargs['to_mask'])
+ flagged = isflagged(flagger[field], kwargs['to_mask'])
# drop all points that hold no relevant grid information
datcol = datcol[~flagged].dropna()
@@ -286,12 +282,15 @@ def interpolateIndex(
# flags reshaping
flagscol = flagscol[~flagged]
- flagscol = _overlap_rs(flagscol, freq, UNFLAGGED)
+ flagscol = _resampleOverlapping(flagscol, freq, UNFLAGGED)
+ dummy = pd.Series(UNTOUCHED, index=data[field].index, dtype=float)
+
+ # do the reshaping on the history
flagger = applyFunctionOnHistory(
flagger, field,
- hist_func=_overlap_rs, hist_kws=dict(freq=freq, fill_value=UNFLAGGED),
- mask_func=_overlap_rs, mask_kws=dict(freq=freq, fill_value=False),
- last_column=flagscol
+ hist_func=_resampleOverlapping, hist_kws=dict(freq=freq, fill_value=UNFLAGGED),
+ mask_func=_resampleOverlapping, mask_kws=dict(freq=freq, fill_value=False),
+ last_column='dummy'
)
return data, flagger
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 40e2bd50f..482dd75c3 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -568,33 +568,43 @@ def resample(
return data, flagger
-def _getChunkBounds(target_datcol, flagscol, freq):
- chunk_end = target_datcol.reindex(flagscol.index, method='bfill', tolerance=freq)
- chunk_start = target_datcol.reindex(flagscol.index, method='ffill', tolerance=freq)
+def _getChunkBounds(target: pd.Series, flagscol: pd.Series, freq: str):
+ chunk_end = target.reindex(flagscol.index, method='bfill', tolerance=freq)
+ chunk_start = target.reindex(flagscol.index, method='ffill', tolerance=freq)
ignore_flags = (chunk_end.isna() | chunk_start.isna())
return ignore_flags
-def _inverseInterpolation(source_col, freq=None, chunk_bounds=None, target_flagscol=None):
- source_col = source_col.copy()
+def _inverseInterpolation(source: pd.Series, target: pd.Series, freq: str, chunk_bounds) -> pd.Series:
+ """
+ Do a inverse interpolation.
+ """
+ source = source.copy()
if len(chunk_bounds) > 0:
- source_col[chunk_bounds] = np.nan
- backprojected = source_col.reindex(target_flagscol.index, method="bfill", tolerance=freq)
- fwrdprojected = source_col.reindex(target_flagscol.index, method="ffill", tolerance=freq)
+ source[chunk_bounds] = np.nan
+ backprojected = source.reindex(target.index, method="bfill", tolerance=freq)
+ fwrdprojected = source.reindex(target.index, method="ffill", tolerance=freq)
return pd.concat([backprojected, fwrdprojected], axis=1).max(axis=1)
-def _inverseAggregation(source_col, freq=None, method=None, target_flagscol=None):
- return source_col.reindex(target_flagscol.index, method=method, tolerance=freq)
+def _inverseAggregation(
+ source: Union[pd.Series, pd.DataFrame],
+ target: Union[pd.Series, pd.DataFrame],
+ freq: str,
+ method: str,
+):
+ return source.reindex(target.index, method=method, tolerance=freq)
+def _inverseShift(source: pd.Series, target: pd.Series, drop_mask: pd.Series,
+ freq: str, method: str, fill_value) -> pd.Series:
+ dtype = source.dtype
-def _inverseShift(source_col, freq=None, method=None, drop_mask=None, target_flagscol=None):
- target_flagscol_drops = target_flagscol[drop_mask]
- target_flagscol = target_flagscol.drop(drop_mask[drop_mask].index)
+ target_drops = target[drop_mask]
+ target = target[~drop_mask]
flags_merged = pd.merge_asof(
- source_col,
- pd.Series(target_flagscol.index.values, index=target_flagscol.index, name="pre_index"),
+ source,
+ target.index.to_series(name='pre_index'),
left_index=True,
right_index=True,
tolerance=freq,
@@ -602,13 +612,13 @@ def _inverseShift(source_col, freq=None, method=None, drop_mask=None, target_fla
)
flags_merged.dropna(subset=["pre_index"], inplace=True)
flags_merged = flags_merged.set_index(["pre_index"]).squeeze()
- target_flagscol[flags_merged.index] = flags_merged.values
+ target[flags_merged.index] = flags_merged.values
# reinsert drops
- source_col = target_flagscol.reindex(target_flagscol.index.join(target_flagscol_drops.index, how="outer"))
- source_col.loc[target_flagscol_drops.index] = target_flagscol_drops.values
+ source = target.reindex(target.index.union(target_drops.index))
+ source.loc[target_drops.index] = target_drops.values
- return source_col
+ return source.fillna(fill_value).astype(dtype, copy=False)
@register(masking='none', module="resampling")
@@ -689,27 +699,33 @@ def reindexFlags(
target_datcol = data[field]
target_flagscol = flagger[field]
- blank_dummy = pd.Series(np.nan, target_flagscol.index)
+ dummy = pd.Series(np.nan, target_flagscol.index)
+
if method[-13:] == "interpolation":
ignore = _getChunkBounds(target_datcol, flagscol, freq)
- merge_func = _inverseInterpolation
- merge_dict = dict(freq=freq, chunk_bounds=ignore, target_flagscol=blank_dummy)
- mask_dict = {**merge_dict, 'chunk_bounds':[]}
+ func = _inverseInterpolation
+ func_kws = dict(freq=freq, chunk_bounds=ignore, target=dummy)
+ mask_kws = {**func_kws, 'chunk_bounds': []}
- if method[-3:] == "agg" or method == "match":
+ elif method[-3:] == "agg" or method == "match":
projection_method = METHOD2ARGS[method][0]
tolerance = METHOD2ARGS[method][1](freq)
- merge_func = _inverseAggregation
- merge_dict = mask_dict = dict(freq=tolerance, method=projection_method, target_flagscol=blank_dummy)
+ func = _inverseAggregation
+ func_kws = dict(freq=tolerance, method=projection_method, target=dummy)
+ mask_kws = func_kws
- if method[-5:] == "shift":
+ elif method[-5:] == "shift":
drop_mask = (target_datcol.isna() | isflagged(target_flagscol, kwargs['to_mask']))
projection_method = METHOD2ARGS[method][0]
tolerance = METHOD2ARGS[method][1](freq)
- merge_func = _inverseShift
- merge_dict = mask_dict = dict(freq=tolerance, method=projection_method, drop_mask=drop_mask, target_flagscol=blank_dummy)
+ func = _inverseShift
+ kws = dict(freq=tolerance, method=projection_method, drop_mask=drop_mask, target=dummy)
+ func_kws = {**kws, 'fill_value': UNTOUCHED}
+ mask_kws = {**kws, 'fill_value': False}
+
+ else:
+ raise ValueError(f"unknown method {method}")
- tmp_flagger = applyFunctionOnHistory(flagger, source, merge_func, merge_dict, merge_func, mask_dict,
- last_column=blank_dummy)
+ tmp_flagger = applyFunctionOnHistory(flagger, source, func, func_kws, func, mask_kws, last_column=dummy)
flagger = appendHistory(flagger, field, tmp_flagger.history[source])
return data, flagger
diff --git a/saqc/lib/ts_operators.py b/saqc/lib/ts_operators.py
index de9de79d2..37d8253ab 100644
--- a/saqc/lib/ts_operators.py
+++ b/saqc/lib/ts_operators.py
@@ -7,6 +7,7 @@ The module gathers all kinds of timeseries tranformations.
import logging
import re
+from typing import Union
import pandas as pd
import numpy as np
@@ -252,12 +253,13 @@ def interpolateNANs(data, method, order=2, inter_limit=2, downgrade_interpolatio
def aggregate2Freq(
- data, method, freq, agg_func, fill_value=np.nan, max_invalid_total=None, max_invalid_consec=None
+ data: pd.Series, method, freq, agg_func, fill_value=np.nan, max_invalid_total=None, max_invalid_consec=None
):
- # The function aggregates values to an equidistant frequency grid with agg_func.
- # Timestamps that have no values projected on them, get "fill_value" assigned. Also,
- # "fill_value" serves as replacement for "invalid" intervals
-
+ """
+ The function aggregates values to an equidistant frequency grid with agg_func.
+ Timestamps that gets no values projected, get filled with the fill-value. It
+ also serves as a replacement for "invalid" intervals.
+ """
methods = {
"nagg": lambda seconds_total: (seconds_total/2, "left", "left"),
"bagg": lambda _: (0, "left", "left"),
@@ -309,9 +311,11 @@ def aggregate2Freq(
return data
-def shift2Freq(data, method, freq, fill_value=np.nan):
- # shift timestamps backwards/forwards in order to allign them with an equidistant
- # frequencie grid.
+def shift2Freq(data: Union[pd.Series, pd.DataFrame], method: str, freq: str, fill_value):
+ """
+ shift timestamps backwards/forwards in order to align them with an equidistant
+ frequency grid. Resulting Nan's are replaced with the fill-value.
+ """
methods = {
"fshift": lambda freq: ("ffill", pd.Timedelta(freq)),
diff --git a/tests/funcs/test_modelling.py b/tests/funcs/test_modelling.py
index 23cc82ab2..248c12246 100644
--- a/tests/funcs/test_modelling.py
+++ b/tests/funcs/test_modelling.py
@@ -6,6 +6,7 @@
import dios
+from saqc import BAD, UNFLAGGED
from saqc.flagger import initFlagsLike
from saqc.funcs.tools import mask
from saqc.funcs.residues import calculatePolynomialResidues, calculateRollingResidues
@@ -46,23 +47,35 @@ def test_modelling_mask(dat):
data, _ = dat()
data = dios.DictOfSeries(data)
flagger = initFlagsLike(data)
- data_seasonal, flagger_seasonal = mask(data, "data", flagger, mode='periodic', period_start="20:00",
- period_end="40:00", include_bounds=False)
- flaggs = flagger_seasonal["data"]
- assert flaggs[np.logical_and(20 <= flaggs.index.minute, 40 >= flaggs.index.minute)].isna().all()
- data_seasonal, flagger_seasonal = mask(data, "data", flagger, mode='periodic', period_start="15:00:00",
- period_end="02:00:00")
- flaggs = flagger_seasonal["data"]
- assert flaggs[np.logical_and(15 <= flaggs.index.hour, 2 >= flaggs.index.hour)].isna().all()
- data_seasonal, flagger_seasonal = mask(data, "data", flagger, mode='periodic', period_start="03T00:00:00",
- period_end="10T00:00:00")
- flaggs = flagger_seasonal["data"]
- assert flaggs[np.logical_and(3 <= flaggs.index.hour, 10 >= flaggs.index.hour)].isna().all()
+ field = "data"
+
+ # set flags everywhere to test unflagging
+ flagger[:, field] = BAD
+
+ common = dict(data=data, field=field, flagger=flagger, mode='periodic')
+ data_seasonal, flagger_seasonal = mask(**common, period_start="20:00", period_end="40:00", include_bounds=False)
+ flags = flagger_seasonal[field]
+ m = (20 <= flags.index.minute) & (flags.index.minute <= 40)
+ assert all(flagger_seasonal[field][m] == UNFLAGGED)
+ assert all(data_seasonal[field][m].isna())
+
+ data_seasonal, flagger_seasonal = mask(**common, period_start="15:00:00", period_end="02:00:00")
+ flags = flagger_seasonal[field]
+ m = (15 <= flags.index.hour) & (flags.index.hour <= 2)
+ assert all(flagger_seasonal[field][m] == UNFLAGGED)
+ assert all(data_seasonal[field][m].isna())
+
+ data_seasonal, flagger_seasonal = mask(**common, period_start="03T00:00:00", period_end="10T00:00:00")
+ flags = flagger_seasonal[field]
+ m = (3 <= flags.index.hour) & (flags.index.hour <= 10)
+ assert all(flagger_seasonal[field][m] == UNFLAGGED)
+ assert all(data_seasonal[field][m].isna())
mask_ser = pd.Series(False, index=data["data"].index)
mask_ser[::5] = True
data["mask_ser"] = mask_ser
flagger = initFlagsLike(data)
data_masked, flagger_masked = mask(data, "data", flagger, mode='mask_var', mask_var="mask_ser")
- flaggs = flagger_masked["data"]
- assert flaggs[data_masked['mask_ser']].isna().all()
+ m = mask_ser
+ assert all(flagger_masked[field][m] == UNFLAGGED)
+ assert all(data_masked[field][m].isna())
--
GitLab
From cf4645b28ea1b17672dcf7a0174f393ac10324d5 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 23 Mar 2021 01:57:26 +0100
Subject: [PATCH 075/180] removed to_drop and other unused params,
auto-reformat files, added TODOS
---
saqc/flagger/flags.py | 2 +-
saqc/funcs/breaks.py | 6 +-
saqc/funcs/changepoints.py | 46 ++++++------
saqc/funcs/constants.py | 26 ++++---
saqc/funcs/curvefit.py | 18 +++--
saqc/funcs/drift.py | 63 +++++++----------
saqc/funcs/interpolation.py | 10 +--
saqc/funcs/outliers.py | 100 +++++++++++++-------------
saqc/funcs/resampling.py | 131 ++++++++++++++---------------------
saqc/funcs/residues.py | 17 +++--
saqc/funcs/rolling.py | 88 +++++++++++------------
saqc/funcs/scores.py | 21 +++---
saqc/funcs/transformation.py | 5 +-
saqc/lib/tools.py | 12 ----
14 files changed, 242 insertions(+), 303 deletions(-)
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index 28a0cef59..d40544a95 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -387,7 +387,7 @@ def applyFunctionOnHistory(
new_history.mask.iloc[:, -1:] = True
else:
if isinstance(last_column, str) and last_column == 'dummy':
- last_column = pd.Series(UNTOUCHED, index=new_history.mask.index, dtype=float)
+ last_column = pd.Series(UNTOUCHED, index=new_history.index, dtype=float)
new_history.append(last_column, force=True)
diff --git a/saqc/funcs/breaks.py b/saqc/funcs/breaks.py
index 107c3c3e7..6c394e3e7 100644
--- a/saqc/funcs/breaks.py
+++ b/saqc/funcs/breaks.py
@@ -53,7 +53,6 @@ def flagMissing(
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
"""
-
datacol = data[field]
if np.isnan(nodata):
mask = datacol.isna()
@@ -114,7 +113,6 @@ def flagIsolated(
--------
:py:func:`flagMissing`
"""
-
gap_window = pd.tseries.frequencies.to_offset(gap_window)
group_window = pd.tseries.frequencies.to_offset(group_window)
@@ -166,8 +164,7 @@ def flagJumps(
Minimum number of periods that have to be present in a window of size `winsz`, so that
the mean value obtained from that window is regarded valid.
"""
-
- data, flagger = assignChangePointCluster(
+ return assignChangePointCluster(
data, field, flagger,
stat_func=lambda x, y: np.abs(np.mean(x) - np.mean(y)),
thresh_func=lambda x, y: thresh,
@@ -179,4 +176,3 @@ def flagJumps(
**kwargs
)
- return data, flagger
diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py
index 200711da4..7025ad712 100644
--- a/saqc/funcs/changepoints.py
+++ b/saqc/funcs/changepoints.py
@@ -27,12 +27,12 @@ def flagChangePoints(
thresh_func: Callable[[np.ndarray, np.ndarray], float],
bwd_window: FreqString,
min_periods_bwd: IntegerWindow,
- fwd_window: Optional[FreqString]=None,
- min_periods_fwd: Optional[IntegerWindow]=None,
- closed: Literal["right", "left", "both", "neither"]="both",
- try_to_jit: bool=True, # TODO rm, not a user decision
- reduce_window: FreqString=None,
- reduce_func: Callable[[np.ndarray, np.ndarray], int]=lambda x, _: x.argmax(),
+ fwd_window: Optional[FreqString] = None,
+ min_periods_fwd: Optional[IntegerWindow] = None,
+ closed: Literal["right", "left", "both", "neither"] = "both",
+ try_to_jit: bool = True, # TODO rm, not a user decision
+ reduce_window: FreqString = None,
+ reduce_func: Callable[[np.ndarray, np.ndarray], int] = lambda x, _: x.argmax(),
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -85,8 +85,7 @@ def flagChangePoints(
-------
"""
-
- data, flagger = assignChangePointCluster(
+ return assignChangePointCluster(
data, field, flagger, stat_func=stat_func, thresh_func=thresh_func,
bwd_window=bwd_window, min_periods_bwd=min_periods_bwd,
fwd_window=fwd_window, min_periods_fwd=min_periods_fwd, closed=closed,
@@ -94,7 +93,6 @@ def flagChangePoints(
reduce_func=reduce_func, flag_changepoints=True, model_by_resids=False,
assign_cluster=False, **kwargs
)
- return data, flagger
@register(masking='field', module="changepoints")
@@ -104,18 +102,17 @@ def assignChangePointCluster(
thresh_func: Callable[[np.array, np.array], float],
bwd_window: str,
min_periods_bwd: int,
- fwd_window: str=None,
- min_periods_fwd: Optional[int]=None,
- closed: Literal["right", "left", "both", "neither"]="both",
- try_to_jit: bool=True, # TODO: rm, not a user decision
- reduce_window: str=None,
- reduce_func: Callable[[np.ndarray, np.ndarray], float]=lambda x, _: x.argmax(),
- model_by_resids: bool=False,
- flag_changepoints: bool=False,
- assign_cluster: bool=True,
+ fwd_window: str = None,
+ min_periods_fwd: Optional[int] = None,
+ closed: Literal["right", "left", "both", "neither"] = "both",
+ try_to_jit: bool = True, # TODO: rm, not a user decision
+ reduce_window: str = None,
+ reduce_func: Callable[[np.ndarray, np.ndarray], float] = lambda x, _: x.argmax(),
+ model_by_resids: bool = False,
+ flag_changepoints: bool = False,
+ assign_cluster: bool = True,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
-
"""
Assigns label to the data, aiming to reflect continous regimes of the processes the data is assumed to be
generated by.
@@ -209,10 +206,13 @@ def assignChangePointCluster(
try_to_jit = False
logging.warning('Could not jit passed statistic - omitting jitting!')
+ args = data_arr, bwd_start, fwd_end, split, stat_func, thresh_func, check_len
+
if try_to_jit:
- stat_arr, thresh_arr = _slidingWindowSearchNumba(data_arr, bwd_start, fwd_end, split, stat_func, thresh_func, check_len)
+ stat_arr, thresh_arr = _slidingWindowSearchNumba(*args)
else:
- stat_arr, thresh_arr = _slidingWindowSearch(data_arr, bwd_start, fwd_end, split, stat_func, thresh_func, check_len)
+ stat_arr, thresh_arr = _slidingWindowSearch(*args)
+
result_arr = stat_arr > thresh_arr
if model_by_resids:
@@ -251,7 +251,7 @@ def assignChangePointCluster(
def _slidingWindowSearchNumba(data_arr, bwd_start, fwd_end, split, stat_func, thresh_func, num_val):
stat_arr = np.zeros(num_val)
thresh_arr = np.zeros(num_val)
- for win_i in numba.prange(0, num_val-1):
+ for win_i in numba.prange(0, num_val - 1):
x = data_arr[bwd_start[win_i]:split[win_i]]
y = data_arr[split[win_i]:fwd_end[win_i]]
stat_arr[win_i] = stat_func(x, y)
@@ -262,7 +262,7 @@ def _slidingWindowSearchNumba(data_arr, bwd_start, fwd_end, split, stat_func, th
def _slidingWindowSearch(data_arr, bwd_start, fwd_end, split, stat_func, thresh_func, num_val):
stat_arr = np.zeros(num_val)
thresh_arr = np.zeros(num_val)
- for win_i in range(0, num_val-1):
+ for win_i in range(0, num_val - 1):
x = data_arr[bwd_start[win_i]:split[win_i]]
y = data_arr[split[win_i]:fwd_end[win_i]]
stat_arr[win_i] = stat_func(x, y)
diff --git a/saqc/funcs/constants.py b/saqc/funcs/constants.py
index 34392fd96..02327498f 100644
--- a/saqc/funcs/constants.py
+++ b/saqc/funcs/constants.py
@@ -41,7 +41,7 @@ def flagConstants(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
field : str
- The fieldname of the column, holding the data-to-be-flagged.
+ Name of the column, holding the data-to-be-flagged.
flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
thresh : float
@@ -57,10 +57,9 @@ def flagConstants(
The flagger object, holding flags and additional informations related to `data`.
Flags values may have changed, relatively to the flagger input.
"""
-
- d = data[field]
if not isinstance(window, str):
raise TypeError('window must be offset string.')
+ d = data[field]
# min_periods=2 ensures that at least two non-nan values are present
# in each window and also min() == max() == d[i] is not possible.
@@ -89,7 +88,6 @@ def flagByVariance(
max_consec_missing: int=None,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
-
"""
Function flags plateaus/series of constant values. Any interval of values y(t),..y(t+n) is flagged, if:
@@ -125,21 +123,27 @@ def flagByVariance(
The flagger object, holding flags and additional informations related to `data`.
Flags values may have changed, relatively to the flagger input.
"""
-
dataseries = data[field]
- data_rate = getFreqDelta(dataseries.index)
- if not data_rate:
+ delta = getFreqDelta(dataseries.index)
+ if not delta:
raise IndexError('Timeseries irregularly sampled!')
+
if max_missing is None:
max_missing = np.inf
+
if max_consec_missing is None:
max_consec_missing = np.inf
- min_periods = int(np.ceil(pd.Timedelta(window) / pd.Timedelta(data_rate)))
- plateaus = dataseries.rolling(window=window, min_periods=min_periods).apply(
- lambda x: True if varQC(x, max_missing, max_consec_missing) <= thresh else np.nan, raw=False,
- )
+ min_periods = int(np.ceil(pd.Timedelta(window) / pd.Timedelta(delta)))
+
+ def var_below_thresh(s: pd.Series):
+ if varQC(s, max_missing, max_consec_missing) <= thresh:
+ return True
+ return np.nan
+
+ rolling = dataseries.rolling(window=window, min_periods=min_periods)
+ plateaus = rolling.apply(var_below_thresh, raw=False)
# are there any candidates for beeing flagged plateau-ish
if plateaus.sum() == 0:
diff --git a/saqc/funcs/curvefit.py b/saqc/funcs/curvefit.py
index 9623d49d1..fc04cbeac 100644
--- a/saqc/funcs/curvefit.py
+++ b/saqc/funcs/curvefit.py
@@ -8,24 +8,24 @@ from typing_extensions import Literal
import numpy as np
import pandas as pd
-
from dios import DictOfSeries
from saqc.core.register import register
from saqc.lib.tools import getFreqDelta
from saqc.flagger import Flagger
-from saqc.lib.ts_operators import polyRollerIrregular, polyRollerNumba, polyRoller, polyRollerNoMissingNumba, polyRollerNoMissing
+from saqc.lib.ts_operators import polyRollerIrregular, polyRollerNumba, polyRoller, polyRollerNoMissingNumba, \
+ polyRollerNoMissing
@register(masking='field', module="curvefit")
def fitPolynomial(data: DictOfSeries, field: str, flagger: Flagger,
winsz: Union[int, str],
polydeg: int,
- numba: Literal[True, False, "auto"]="auto",
- eval_flags: bool=True,
- min_periods: int=0,
- return_residues: bool=False,
+ numba: Literal[True, False, "auto"] = "auto",
+ eval_flags: bool = True,
+ min_periods: int = 0,
+ return_residues: bool = False,
**kwargs) -> Tuple[DictOfSeries, Flagger]:
"""
Function fits a polynomial model to the data and returns the fitted data curve.
@@ -100,7 +100,6 @@ def fitPolynomial(data: DictOfSeries, field: str, flagger: Flagger,
flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
-
"""
# TODO: some (rater large) parts are functional similar to saqc.funcs.rolling.roll
if data[field].empty:
@@ -127,8 +126,8 @@ def fitPolynomial(data: DictOfSeries, field: str, flagger: Flagger,
temp = residues.copy()
for k in centers_iloc.iteritems():
residues.iloc[k[1]] = temp[k[0]]
- residues[residues.index[0] : residues.index[centers_iloc[0]]] = np.nan
- residues[residues.index[centers_iloc[-1]] : residues.index[-1]] = np.nan
+ residues[residues.index[0]: residues.index[centers_iloc[0]]] = np.nan
+ residues[residues.index[centers_iloc[-1]]: residues.index[-1]] = np.nan
else:
if isinstance(winsz, str):
winsz = pd.Timedelta(winsz) // regular
@@ -200,4 +199,3 @@ def fitPolynomial(data: DictOfSeries, field: str, flagger: Flagger,
flagger[field] = worst
return data, flagger
-
diff --git a/saqc/funcs/drift.py b/saqc/funcs/drift.py
index ea1a4d20c..14417c3f0 100644
--- a/saqc/funcs/drift.py
+++ b/saqc/funcs/drift.py
@@ -21,7 +21,6 @@ from saqc.lib.tools import detectDeviants
from saqc.lib.types import FreqString, ColumnName, CurveFitter, TimestampColumnName
from saqc.lib.ts_operators import expModelFunc
-
LinkageString = Literal["single", "complete", "average", "weighted", "centroid", "median", "ward"]
@@ -33,9 +32,9 @@ def flagDriftFromNorm(
fields: Sequence[ColumnName],
segment_freq: FreqString,
norm_spread: float,
- norm_frac: float=0.5,
- metric: Callable[[np.ndarray, np.ndarray], float]=lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
- linkage_method: LinkageString="single",
+ norm_frac: float = 0.5,
+ metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
+ linkage_method: LinkageString = "single",
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -149,7 +148,7 @@ def flagDriftFromReference(
fields: Sequence[ColumnName],
segment_freq: FreqString,
thresh: float,
- metric: Callable[[np.ndarray, np.ndarray], float]=lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
+ metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -193,7 +192,6 @@ def flagDriftFromReference(
That is, why, the "averaged manhatten metric" is set as the metric default, since it corresponds to the
averaged value distance, two timeseries have (as opposed by euclidean, for example).
"""
-
data_to_flag = data[fields].to_df()
data_to_flag.dropna(inplace=True)
@@ -227,13 +225,11 @@ def flagDriftFromScaledNorm(
fields_scale2: Sequence[ColumnName],
segment_freq: FreqString,
norm_spread: float,
- norm_frac: float=0.5,
- metric: Callable[[np.ndarray, np.ndarray], float]=lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
- linkage_method: LinkageString="single",
+ norm_frac: float = 0.5,
+ metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
+ linkage_method: LinkageString = "single",
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
-
-
"""
The function linearly rescales one set of variables to another set of variables with a different scale and then
flags value courses that significantly deviate from a group of normal value courses.
@@ -298,7 +294,6 @@ def flagDriftFromScaledNorm(
Introduction to Hierarchical clustering:
[2] https://en.wikipedia.org/wiki/Hierarchical_clustering
"""
-
fields = list(fields_scale1) + list(fields_scale2)
data_to_flag = data[fields].to_df()
data_to_flag.dropna(inplace=True)
@@ -343,8 +338,8 @@ def correctExponentialDrift(
field: ColumnName,
flagger: Flagger,
maint_data_field: ColumnName,
- cal_mean: int=5,
- flag_maint_period: bool=False,
+ cal_mean: int = 5,
+ flag_maint_period: bool = False,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -420,7 +415,7 @@ def correctExponentialDrift(
for k in range(0, maint_data.shape[0] - 1):
# assign group numbers for the timespans in between one maintenance ending and the beginning of the next
# maintenance time itself remains np.nan assigned
- drift_frame.loc[maint_data.values[k] : pd.Timestamp(maint_data.index[k + 1]), "drift_group"] = k
+ drift_frame.loc[maint_data.values[k]: pd.Timestamp(maint_data.index[k + 1]), "drift_group"] = k
# define target values for correction
drift_grouper = drift_frame.groupby("drift_group")
@@ -453,8 +448,8 @@ def correctRegimeAnomaly(
flagger: Flagger,
cluster_field: ColumnName,
model: CurveFitter,
- regime_transmission: Optional[FreqString]=None,
- x_date: bool=False,
+ regime_transmission: Optional[FreqString] = None,
+ x_date: bool = False,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -501,7 +496,6 @@ def correctRegimeAnomaly(
flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
"""
-
cluster_ser = data[cluster_field]
unique_successive = pd.unique(cluster_ser.values)
data_ser = data[field]
@@ -515,10 +509,10 @@ def correctRegimeAnomaly(
for label, regime in regimes:
if x_date is False:
# get seconds data:
- xdata = (regime.index - regime.index[0]).to_numpy(dtype=float)*10**(-9)
+ xdata = (regime.index - regime.index[0]).to_numpy(dtype=float) * 10 ** (-9)
else:
# get seconds from epoch data
- xdata = regime.index.to_numpy(dtype=float)*10**(-9)
+ xdata = regime.index.to_numpy(dtype=float) * 10 ** (-9)
ydata = regime.values
valid_mask = ~np.isnan(ydata)
if regime_transmission is not None:
@@ -533,7 +527,8 @@ def correctRegimeAnomaly(
x_mask[label] = valid_mask
first_normal = unique_successive > 0
- first_valid = np.array([~pd.isna(para_dict[unique_successive[i]]).any() for i in range(0, unique_successive.shape[0])])
+ first_valid = np.array(
+ [~pd.isna(para_dict[unique_successive[i]]).any() for i in range(0, unique_successive.shape[0])])
first_valid = np.where(first_normal & first_valid)[0][0]
last_valid = 1
@@ -543,7 +538,7 @@ def correctRegimeAnomaly(
xdata = x_dict[unique_successive[k]]
ypara = para_dict[unique_successive[k]]
if k > 0:
- target_para = para_dict[unique_successive[k-last_valid]]
+ target_para = para_dict[unique_successive[k - last_valid]]
else:
# first regime has no "last valid" to its left, so we use first valid to the right:
target_para = para_dict[unique_successive[k + first_valid]]
@@ -569,11 +564,10 @@ def correctOffset(
normal_spread: float,
search_winsz: FreqString,
min_periods: int,
- regime_transmission: Optional[FreqString]=None,
+ regime_transmission: Optional[FreqString] = None,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
-
Parameters
----------
data : dios.DictOfSeries
@@ -608,7 +602,6 @@ def correctOffset(
The flagger object, holding flags and additional Informations related to `data`.
"""
-
data, flagger = copy(data, field, flagger, field + '_CPcluster')
data, flagger = assignChangePointCluster(
data, field + '_CPcluster', flagger,
@@ -663,9 +656,9 @@ def flagRegimeAnomaly(
flagger: Flagger,
cluster_field: ColumnName,
norm_spread: float,
- linkage_method: LinkageString="single",
- metric: Callable[[np.ndarray, np.ndarray], float]=lambda x, y: np.abs(np.nanmean(x) - np.nanmean(y)),
- norm_frac: float=0.5,
+ linkage_method: LinkageString = "single",
+ metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: np.abs(np.nanmean(x) - np.nanmean(y)),
+ norm_frac: float = 0.5,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -711,9 +704,7 @@ def flagRegimeAnomaly(
flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional informations related to `data`.
Flags values may have changed, relatively to the flagger input.
-
"""
-
return assignRegimeAnomaly(
data, field, flagger,
cluster_field,
@@ -734,11 +725,11 @@ def assignRegimeAnomaly(
flagger: Flagger,
cluster_field: ColumnName,
norm_spread: float,
- linkage_method: LinkageString="single",
- metric: Callable[[np.array, np.array], float]=lambda x, y: np.abs(np.nanmean(x) - np.nanmean(y)),
- norm_frac: float=0.5,
- set_cluster: bool=True,
- set_flags: bool=False,
+ linkage_method: LinkageString = "single",
+ metric: Callable[[np.array, np.array], float] = lambda x, y: np.abs(np.nanmean(x) - np.nanmean(y)),
+ norm_frac: float = 0.5,
+ set_cluster: bool = True,
+ set_flags: bool = False,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -793,9 +784,7 @@ def assignRegimeAnomaly(
flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional informations related to `data`.
Flags values may have changed, relatively to the flagger input.
-
"""
-
series = data[cluster_field]
cluster = np.unique(series)
cluster_dios = DictOfSeries({i: data[field][series == i] for i in cluster})
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index be93d5492..c5d4f0768 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -13,8 +13,6 @@ from saqc.constants import *
from saqc.core.register import register, isflagged
from saqc.flagger import Flagger
from saqc.flagger.flags import applyFunctionOnHistory
-
-from saqc.lib.tools import toSequence, evalFreqStr, getDropMask
from saqc.lib.ts_operators import interpolateNANs
_SUPPORTED_METHODS = Literal[
@@ -172,6 +170,7 @@ def interpolateInvalid(
def _resampleOverlapping(data: pd.Series, freq: str, fill_value):
+ """TODO: docstring needed"""
dtype = data.dtype
end = data.index[-1].ceil(freq)
data = data.resample(freq).max()
@@ -245,7 +244,6 @@ def interpolateIndex(
return data, flagger
datcol = data[field].copy()
- flagscol = flagger[field]
start, end = datcol.index[0].floor(freq), datcol.index[-1].ceil(freq)
grid_index = pd.date_range(start=start, end=end, freq=freq, name=datcol.index.name)
@@ -279,12 +277,6 @@ def interpolateIndex(
# store interpolated grid
data[field] = inter_data[grid_index]
- # flags reshaping
- flagscol = flagscol[~flagged]
-
- flagscol = _resampleOverlapping(flagscol, freq, UNFLAGGED)
- dummy = pd.Series(UNTOUCHED, index=data[field].index, dtype=float)
-
# do the reshaping on the history
flagger = applyFunctionOnHistory(
flagger, field,
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index ab486487d..189995cc5 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -29,14 +29,14 @@ import saqc.lib.ts_operators as ts_ops
@register(masking='field', module="outliers")
def flagByStray(
- data: DictOfSeries,
- field: ColumnName,
- flagger: Flagger,
- partition_freq: Optional[Union[IntegerWindow, FreqString]]=None,
- partition_min: int=11,
- iter_start: float=0.5,
- alpha: float=0.05,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flagger: Flagger,
+ partition_freq: Optional[Union[IntegerWindow, FreqString]] = None,
+ partition_min: int = 11,
+ iter_start: float = 0.5,
+ alpha: float = 0.05,
+ **kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
Flag outliers in 1-dimensional (score) data with the STRAY Algorithm.
@@ -132,11 +132,11 @@ def _evalStrayLabels(
field: str,
flagger: Flagger,
fields: Sequence[str],
- reduction_range: Optional[str]=None,
- reduction_drop_flagged: bool=False,
- reduction_thresh: float=3.5,
- reduction_min_periods: int=1,
- at_least_one: bool=True,
+ reduction_range: Optional[str] = None,
+ reduction_drop_flagged: bool = False, # TODO: still a case ?
+ reduction_thresh: float = 3.5,
+ reduction_min_periods: int = 1,
+ at_least_one: bool = True,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -197,7 +197,7 @@ def _evalStrayLabels(
# check, wheather value under test is sufficiently centered:
first = test_slice.first_valid_index()
last = test_slice.last_valid_index()
- min_range = pd.Timedelta(reduction_range)/4
+ min_range = pd.Timedelta(reduction_range) / 4
if pd.Timedelta(index[1] - first) < min_range or pd.Timedelta(last - index[1]) < min_range:
polydeg = 0
@@ -213,11 +213,11 @@ def _evalStrayLabels(
x = (test_slice.index.values.astype(float))
x_0 = x[0]
- x = (x - x_0)/10**12
+ x = (x - x_0) / 10 ** 12
polyfitted = poly.polyfit(y=test_slice.values, x=x, deg=polydeg)
- testval = poly.polyval((float(index[1].to_numpy()) - x_0)/10**12, polyfitted)
+ testval = poly.polyval((float(index[1].to_numpy()) - x_0) / 10 ** 12, polyfitted)
testval = val_frame[var][index[1]] - testval
resids = test_slice.values - poly.polyval(x, polyfitted)
@@ -316,7 +316,7 @@ def _expFit(val_frame, scoring_method="kNNMaxGap", n_neighbors=10, iter_start=0.
upper_tail_index = int(np.floor(0.5 * hist_argmax + 0.5 * iter_max_bin_index))
resids_tail_index = findIndex(resids, binz[upper_tail_index], 0)
upper_tail_hist, bins = np.histogram(
- resids[resids_tail_index:iter_index], bins=binz[upper_tail_index : iter_max_bin_index + 1]
+ resids[resids_tail_index:iter_index], bins=binz[upper_tail_index: iter_max_bin_index + 1]
)
while (test_val < crit_val) & (iter_index < resids.size - 1):
@@ -331,7 +331,7 @@ def _expFit(val_frame, scoring_method="kNNMaxGap", n_neighbors=10, iter_start=0.
upper_tail_hist[-1] += 1
iter_max_bin_index = new_iter_max_bin_index
upper_tail_index_new = int(np.floor(0.5 * hist_argmax + 0.5 * iter_max_bin_index))
- upper_tail_hist = upper_tail_hist[upper_tail_index_new - upper_tail_index :]
+ upper_tail_hist = upper_tail_hist[upper_tail_index_new - upper_tail_index:]
upper_tail_index = upper_tail_index_new
# fitting
@@ -355,18 +355,18 @@ def flagMVScores(
field: ColumnName,
flagger: Flagger,
fields: Sequence[ColumnName],
- trafo: Callable[[pd.Series], pd.Series]=lambda x: x,
- alpha: float=0.05,
- n_neighbors: int=10,
- scoring_func: Callable[[pd.Series], float]=np.sum,
- iter_start: float=0.5,
- stray_partition: Optional[Union[IntegerWindow, FreqString]]=None,
- stray_partition_min: int=11,
- trafo_on_partition: bool=True,
- reduction_range: Optional[FreqString]=None,
- reduction_drop_flagged: bool=False,
- reduction_thresh: float=3.5,
- reduction_min_periods: int=1,
+ trafo: Callable[[pd.Series], pd.Series] = lambda x: x,
+ alpha: float = 0.05,
+ n_neighbors: int = 10,
+ scoring_func: Callable[[pd.Series], float] = np.sum,
+ iter_start: float = 0.5,
+ stray_partition: Optional[Union[IntegerWindow, FreqString]] = None,
+ stray_partition_min: int = 11,
+ trafo_on_partition: bool = True,
+ reduction_range: Optional[FreqString] = None,
+ reduction_drop_flagged: bool = False, # TODO: still a case ?
+ reduction_thresh: float = 3.5,
+ reduction_min_periods: int = 1,
**kwargs,
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -509,11 +509,11 @@ def flagRaise(
thresh: float,
raise_window: FreqString,
intended_freq: FreqString,
- average_window: Optional[FreqString]=None,
- mean_raise_factor: float=2.,
- min_slope: Optional[float]=None,
- min_slope_weight: float=0.8,
- numba_boost: bool=True, # TODO: rm, not a user decision
+ average_window: Optional[FreqString] = None,
+ mean_raise_factor: float = 2.,
+ min_slope: Optional[float] = None,
+ min_slope_weight: float = 0.8,
+ numba_boost: bool = True, # TODO: rm, not a user decision
**kwargs,
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -624,23 +624,23 @@ def flagRaise(
# "unflag" values of insufficient deviation to their predecessors
if min_slope is not None:
w_mask = (
- pd.Series(dataseries.index).diff().dt.total_seconds() / intended_freq.total_seconds()
- ) > min_slope_weight
+ pd.Series(dataseries.index).diff().dt.total_seconds() / intended_freq.total_seconds()
+ ) > min_slope_weight
slope_mask = np.abs(dataseries.diff()) < min_slope
to_unflag = raise_series.notna() & w_mask.values & slope_mask
raise_series[to_unflag] = np.nan
# calculate and apply the weighted mean weights (pseudo-harmonization):
weights = (
- pd.Series(dataseries.index).diff(periods=2).shift(-1).dt.total_seconds() / intended_freq.total_seconds() / 2
+ pd.Series(dataseries.index).diff(periods=2).shift(-1).dt.total_seconds() / intended_freq.total_seconds() / 2
)
weights.iloc[0] = 0.5 + (dataseries.index[1] - dataseries.index[0]).total_seconds() / (
- intended_freq.total_seconds() * 2
+ intended_freq.total_seconds() * 2
)
weights.iloc[-1] = 0.5 + (dataseries.index[-1] - dataseries.index[-2]).total_seconds() / (
- intended_freq.total_seconds() * 2
+ intended_freq.total_seconds() * 2
)
weights[weights > 1.5] = 1.5
@@ -669,7 +669,7 @@ def flagRaise(
@register(masking='field', module="outliers")
def flagMAD(
- data: DictOfSeries, field: ColumnName, flagger: Flagger, window: FreqString, z: float=3.5, **kwargs
+ data: DictOfSeries, field: ColumnName, flagger: Flagger, window: FreqString, z: float = 3.5, **kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
The function represents an implementation of the modyfied Z-score outlier detection method.
@@ -733,8 +733,8 @@ def flagOffset(
thresh: float,
tolerance: float,
window: Union[IntegerWindow, FreqString],
- rel_thresh: Optional[float]=None,
- numba_kickin: int=200000, # TODO: rm, not a user decision
+ rel_thresh: Optional[float] = None,
+ numba_kickin: int = 200000, # TODO: rm, not a user decision
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -798,7 +798,7 @@ def flagOffset(
# using reverted series - because ... long story.
ind = dataseries.index
- rev_ind = ind[0] + ((ind[-1]-ind)[::-1])
+ rev_ind = ind[0] + ((ind[-1] - ind)[::-1])
map_i = pd.Series(ind, index=rev_ind)
dataseries = pd.Series(dataseries.values, index=rev_ind)
@@ -887,9 +887,9 @@ def flagByGrubbs(
field: ColumnName,
flagger: Flagger,
winsz: Union[FreqString, IntegerWindow],
- alpha: float=0.05,
- min_periods: int=8,
- check_lagged: bool=False,
+ alpha: float = 0.05,
+ min_periods: int = 8,
+ check_lagged: bool = False,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -992,8 +992,8 @@ def flagRange(
data: DictOfSeries,
field: ColumnName,
flagger: Flagger,
- min: float=-np.inf,
- max: float=np.inf,
+ min: float = -np.inf,
+ max: float = np.inf,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -1035,7 +1035,7 @@ def flagCrossStatistic(
flagger: Flagger,
fields: Sequence[ColumnName],
thresh: float,
- cross_stat: Literal["modZscore", "Zscore"]="modZscore",
+ cross_stat: Literal["modZscore", "Zscore"] = "modZscore",
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 482dd75c3..b5d2a109f 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -15,8 +15,8 @@ from saqc.constants import *
from saqc.core.register import register, isflagged
from saqc.flagger import Flagger, initFlagsLike, History
from saqc.funcs.tools import copy, drop, rename
-from saqc.funcs.interpolation import interpolateIndex
-from saqc.lib.tools import getDropMask, evalFreqStr, getFreqDelta
+from saqc.funcs.interpolation import interpolateIndex, _SUPPORTED_METHODS
+from saqc.lib.tools import evalFreqStr, getFreqDelta
from saqc.lib.ts_operators import shift2Freq, aggregate2Freq
from saqc.flagger.flags import applyFunctionOnHistory, appendHistory
from saqc.lib.rolling import customRoller
@@ -41,9 +41,8 @@ def aggregate(
flagger: Flagger,
freq: str,
value_func,
- flag_func: Callable[[pd.Series], float]=np.nanmax,
- method: Literal["fagg", "bagg", "nagg"]="nagg",
- to_drop: Optional[Union[Any, Sequence[Any]]]=None,
+ flag_func: Callable[[pd.Series], float] = np.nanmax,
+ method: Literal["fagg", "bagg", "nagg"] = "nagg",
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -74,24 +73,26 @@ def aggregate(
----------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
+
field : str
The fieldname of the column, holding the data-to-be-regularized.
+
flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.freq
+
freq : str
The sampling frequency the data is to be aggregated (resampled) at.
+
value_func : Callable
The function you want to use for aggregation.
+
flag_func : Callable
The function you want to aggregate the flags with. It should be capable of operating on the flags dtype
(usually ordered categorical).
+
method : {'fagg', 'bagg', 'nagg'}, default 'nagg'
Specifies which intervals to be aggregated for a certain timestamp. (preceeding, succeeding or
"surrounding" interval). See description above for more details.
- to_drop : {List[str], str}, default None
- Flagtypes you want to drop before aggregation - effectively excluding values that are flagged
- with a flag in to_drop from the aggregation process. Default results in BAD
- values being dropped initially.
Returns
-------
@@ -104,20 +105,9 @@ def aggregate(
"""
data, flagger = copy(data, field, flagger, field + '_original')
- data, flagger = resample(
- data,
- field,
- flagger,
- freq,
- agg_func=value_func,
- flag_agg_func=flag_func,
- method=method,
- empty_intervals_flag=UNFLAGGED,
- to_drop=to_drop,
- all_na_2_empty=True,
- **kwargs,
+ return resample(
+ data, field, flagger, freq=freq, agg_func=value_func, flag_agg_func=flag_func, method=method, **kwargs
)
- return data, flagger
@register(masking='none', module="resampling")
@@ -126,7 +116,6 @@ def linear(
field: str,
flagger: Flagger,
freq: str,
- to_drop: Optional[Union[Any, Sequence[Any]]]=None,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -148,16 +137,15 @@ def linear(
----------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
+
field : str
The fieldname of the column, holding the data-to-be-regularized.
+
flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.freq
+
freq : str
An offset string. The frequency of the grid you want to interpolate your data at.
- to_drop : {List[str], str}, default None
- Flagtypes you want to drop before interpolation - effectively excluding values that are flagged
- with a flag in to_drop from the interpolation process. Default results in BAD
- values being dropped initially.
Returns
-------
@@ -170,10 +158,7 @@ def linear(
"""
data, flagger = copy(data, field, flagger, field + '_original')
- data, flagger = interpolateIndex(
- data, field, flagger, freq, "time", to_drop=to_drop, empty_intervals_flag=UNFLAGGED, **kwargs
- )
- return data, flagger
+ return interpolateIndex(data, field, flagger, freq, "time", **kwargs)
@register(masking='none', module="resampling")
@@ -182,9 +167,8 @@ def interpolate(
field: str,
flagger: Flagger,
freq: str,
- method: Literal["linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric", "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"],
- order: int=1,
- to_drop: Optional[Union[Any, Sequence[Any]]]=None,
+ method: _SUPPORTED_METHODS,
+ order: int = 1,
**kwargs,
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -212,22 +196,23 @@ def interpolate(
----------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
+
field : str
The fieldname of the column, holding the data-to-be-regularized.
+
flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.freq
+
freq : str
An offset string. The frequency of the grid you want to interpolate your data at.
+
method : {"linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric",
- "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"}: string
+ "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"}
The interpolation method you want to apply.
+
order : int, default 1
If your selected interpolation method can be performed at different *orders* - here you pass the desired
order.
- to_drop : {List[str], str}, default None
- Flagtypes you want to drop before interpolation - effectively excluding values that are flagged
- with a flag in `to_drop` from the interpolation process. Default results in ``BAD``
- values being dropped initially.
Returns
-------
@@ -240,18 +225,7 @@ def interpolate(
"""
data, flagger = copy(data, field, flagger, field + '_original')
- data, flagger = interpolateIndex(
- data,
- field,
- flagger,
- freq,
- method=method,
- inter_order=order,
- to_drop=to_drop,
- empty_intervals_flag=UNFLAGGED,
- **kwargs,
- )
- return data, flagger
+ return interpolateIndex(data, field, flagger, freq, method=method, inter_order=order, **kwargs)
@register(masking='none', module="resampling")
@@ -259,8 +233,11 @@ def mapToOriginal(
data: DictOfSeries,
field: str,
flagger: Flagger,
- method: Literal["inverse_fagg", "inverse_bagg", "inverse_nagg", "inverse_fshift", "inverse_bshift", "inverse_nshift", "inverse_interpolation"],
- to_drop: Optional[Union[Any, Sequence[Any]]]=None,
+ method: Literal[
+ "inverse_fagg", "inverse_bagg", "inverse_nagg",
+ "inverse_fshift", "inverse_bshift", "inverse_nshift",
+ "inverse_interpolation"
+ ],
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -305,18 +282,17 @@ def mapToOriginal(
----------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
+
field : str
The fieldname of the column, holding the data-to-be-deharmonized.
+
flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.freq
+
method : {'inverse_fagg', 'inverse_bagg', 'inverse_nagg', 'inverse_fshift', 'inverse_bshift', 'inverse_nshift',
'inverse_interpolation'}
The method used for projection of regularized flags onto original flags. See description above for more
details.
- to_drop : {List[str], str}, default None
- Flagtypes you want to drop before interpolation - effectively excluding values that are flagged
- with a flag in to_drop from the interpolation process. Default results in BAD
- values being dropped initially.
Returns
-------
@@ -327,12 +303,10 @@ def mapToOriginal(
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
-
newfield = str(field) + '_original'
data, flagger = reindexFlags(data, newfield, flagger, method, source=field, to_mask=False)
data, flagger = drop(data, field, flagger)
- data, flagger = rename(data, newfield, flagger, field)
- return data, flagger
+ return rename(data, newfield, flagger, field)
@register(masking='none', module="resampling")
@@ -438,14 +412,14 @@ def resample(
field: str,
flagger: Flagger,
freq: str,
- agg_func: Callable[[pd.Series], pd.Series]=np.mean,
- method: Literal["fagg", "bagg", "nagg"]="bagg",
- max_invalid_total_d: Optional[int]=None,
- max_invalid_consec_d: Optional[int]=None,
- max_invalid_consec_f: Optional[int]=None,
- max_invalid_total_f: Optional[int]=None,
- flag_agg_func: Callable[[pd.Series], float]=max,
- freq_check: Optional[Literal["check", "auto"]]=None,
+ agg_func: Callable[[pd.Series], pd.Series] = np.mean,
+ method: Literal["fagg", "bagg", "nagg"] = "bagg",
+ max_invalid_total_d: Optional[int] = None,
+ max_invalid_consec_d: Optional[int] = None,
+ max_invalid_consec_f: Optional[int] = None,
+ max_invalid_total_f: Optional[int] = None,
+ flag_agg_func: Callable[[pd.Series], float] = max,
+ freq_check: Optional[Literal["check", "auto"]] = None,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -546,8 +520,6 @@ def resample(
max_invalid_consec=max_invalid_consec_d,
)
- dummy = pd.Series(UNTOUCHED, index=datcol.index, dtype=float)
-
kws = dict(
method=method,
freq=freq,
@@ -561,7 +533,7 @@ def resample(
flagger, field,
hist_func=aggregate2Freq, hist_kws=kws,
mask_func=aggregate2Freq, mask_kws=kws,
- last_column=dummy
+ last_column='dummy'
)
data[field] = datcol
@@ -576,9 +548,6 @@ def _getChunkBounds(target: pd.Series, flagscol: pd.Series, freq: str):
def _inverseInterpolation(source: pd.Series, target: pd.Series, freq: str, chunk_bounds) -> pd.Series:
- """
- Do a inverse interpolation.
- """
source = source.copy()
if len(chunk_bounds) > 0:
source[chunk_bounds] = np.nan
@@ -626,7 +595,10 @@ def reindexFlags(
data: DictOfSeries,
field: str,
flagger: Flagger,
- method: Literal["inverse_fagg", "inverse_bagg", "inverse_nagg", "inverse_fshift", "inverse_bshift", "inverse_nshift"],
+ method: Literal[
+ "inverse_fagg", "inverse_bagg", "inverse_nagg",
+ "inverse_fshift", "inverse_bshift", "inverse_nshift"
+ ],
source: str,
freq: Optional[str] = None,
**kwargs
@@ -667,14 +639,19 @@ def reindexFlags(
----------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
+
field : str
The fieldname of the data column, you want to project the source-flags onto.
+
flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional Informations related to `data`.
+
method : {'inverse_fagg', 'inverse_bagg', 'inverse_nagg', 'inverse_fshift', 'inverse_bshift', 'inverse_nshift'}
The method used for projection of source flags onto field flags. See description above for more details.
+
source : str
The source source of flags projection.
+
freq : {None, str},default None
The freq determines the projection range for the projection method. See above description for more details.
Defaultly (None), the sampling frequency of source is used.
@@ -688,8 +665,6 @@ def reindexFlags(
Flags values and shape may have changed relatively to the flagger input.
"""
flagscol = flagger[source]
- if flagscol.empty:
- return data, flagger
if freq is None:
freq = getFreqDelta(flagscol.index)
@@ -719,7 +694,7 @@ def reindexFlags(
projection_method = METHOD2ARGS[method][0]
tolerance = METHOD2ARGS[method][1](freq)
func = _inverseShift
- kws = dict(freq=tolerance, method=projection_method, drop_mask=drop_mask, target=dummy)
+ kws = dict(freq=tolerance, method=projection_method, drop_mask=drop_mask, target=dummy)
func_kws = {**kws, 'fill_value': UNTOUCHED}
mask_kws = {**kws, 'fill_value': False}
diff --git a/saqc/funcs/residues.py b/saqc/funcs/residues.py
index 16684a43f..6abcfd2d6 100644
--- a/saqc/funcs/residues.py
+++ b/saqc/funcs/residues.py
@@ -21,9 +21,9 @@ def calculatePolynomialResidues(
flagger: Flagger,
winsz: Union[str, int],
polydeg: int,
- numba: Literal[True, False, "auto"]="auto",
- eval_flags: bool=True,
- min_periods: Optional[int]=0,
+ numba: Literal[True, False, "auto"] = "auto", # TODO: rm, not a a user decision
+ eval_flags: bool = True, # TODO, not valid anymore, if still needed, maybe assign user-passed ``flag``?
+ min_periods: Optional[int] = 0,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -117,13 +117,13 @@ def calculateRollingResidues(
field: str,
flagger: Flagger,
winsz: Union[str, int],
- func: Callable[[np.ndarray], np.ndarray]=np.mean,
- eval_flags: bool=True,
- min_periods: Optional[int]=0,
- center: bool=True,
+ func: Callable[[np.ndarray], np.ndarray] = np.mean,
+ eval_flags: bool = True,
+ min_periods: Optional[int] = 0,
+ center: bool = True,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
-
+ """ TODO: docstring needed"""
return roll(
data, field, flagger,
winsz=winsz,
@@ -134,4 +134,3 @@ def calculateRollingResidues(
return_residues=True,
**kwargs
)
-
diff --git a/saqc/funcs/rolling.py b/saqc/funcs/rolling.py
index a0740e511..6a40c93c2 100644
--- a/saqc/funcs/rolling.py
+++ b/saqc/funcs/rolling.py
@@ -20,55 +20,55 @@ def roll(
flagger: Flagger,
winsz: Union[str, int],
func: Callable[[pd.Series], float]=np.mean,
- eval_flags: bool=True,
+ eval_flags: bool=True, # TODO: not applicable anymore
min_periods: int=0,
center: bool=True,
- return_residues=False,
+ return_residues=False, # TODO: this should not be public, a wrapper would be better
**kwargs
):
"""
- Models the data with the rolling mean and returns the residues.
-
- Note, that the residues will be stored to the `field` field of the input data, so that the data that is modelled
- gets overridden.
-
- Parameters
- ----------
- data : dios.DictOfSeries
- A dictionary of pandas.Series, holding all the data.
- field : str
- The fieldname of the column, holding the data-to-be-modelled.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
- winsz : {int, str}
- The size of the window you want to roll with. If an integer is passed, the size
- refers to the number of periods for every fitting window. If an offset string is passed,
- the size refers to the total temporal extension.
- For regularly sampled timeseries, the period number will be casted down to an odd number if
- center = True.
- func : Callable[np.array, float], default np.mean
- Function to apply on the rolling window and obtain the curve fit value.
- eval_flags : bool, default True
- Wheather or not to assign new flags to the calculated residuals. If True, a residual gets assigned the worst
- flag present in the interval, the data for its calculation was obtained from.
- Currently not implemented in combination with not-harmonized timeseries.
- min_periods : int, default 0
- The minimum number of periods, that has to be available in every values fitting surrounding for the mean
- fitting to be performed. If there are not enough values, np.nan gets assigned. Default (0) results in fitting
- regardless of the number of values present.
- center : bool, default True
- Wheather or not to center the window the mean is calculated of around the reference value. If False,
- the reference value is placed to the right of the window (classic rolling mean with lag.)
-
- Returns
- -------
- data : dios.DictOfSeries
- A dictionary of pandas.Series, holding all the data.
- Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
- """
+ Models the data with the rolling mean and returns the residues.
+
+ Note, that the residues will be stored to the `field` field of the input data, so that the data that is modelled
+ gets overridden.
+
+ Parameters
+ ----------
+ data : dios.DictOfSeries
+ A dictionary of pandas.Series, holding all the data.
+ field : str
+ The fieldname of the column, holding the data-to-be-modelled.
+ flagger : saqc.flagger.Flagger
+ A flagger object, holding flags and additional Informations related to `data`.
+ winsz : {int, str}
+ The size of the window you want to roll with. If an integer is passed, the size
+ refers to the number of periods for every fitting window. If an offset string is passed,
+ the size refers to the total temporal extension.
+ For regularly sampled timeseries, the period number will be casted down to an odd number if
+ center = True.
+ func : Callable[np.array, float], default np.mean
+ Function to apply on the rolling window and obtain the curve fit value.
+ eval_flags : bool, default True
+ Wheather or not to assign new flags to the calculated residuals. If True, a residual gets assigned the worst
+ flag present in the interval, the data for its calculation was obtained from.
+ Currently not implemented in combination with not-harmonized timeseries.
+ min_periods : int, default 0
+ The minimum number of periods, that has to be available in every values fitting surrounding for the mean
+ fitting to be performed. If there are not enough values, np.nan gets assigned. Default (0) results in fitting
+ regardless of the number of values present.
+ center : bool, default True
+ Wheather or not to center the window the mean is calculated of around the reference value. If False,
+ the reference value is placed to the right of the window (classic rolling mean with lag.)
+
+ Returns
+ -------
+ data : dios.DictOfSeries
+ A dictionary of pandas.Series, holding all the data.
+ Data values may have changed relatively to the data input.
+ flagger : saqc.flagger.Flagger
+ The flagger object, holding flags and additional Informations related to `data`.
+ Flags values may have changed relatively to the flagger input.
+ """
data = data.copy()
to_fit = data[field]
diff --git a/saqc/funcs/scores.py b/saqc/funcs/scores.py
index 1a49f4e1e..1f5d0456e 100644
--- a/saqc/funcs/scores.py
+++ b/saqc/funcs/scores.py
@@ -21,19 +21,20 @@ def assignKNNScore(
field: str,
flagger: Flagger,
fields: Sequence[str],
- n_neighbors: int=10,
- trafo: Callable[[pd.Series], pd.Series]=lambda x: x,
- trafo_on_partition: bool=True,
- scoring_func: Callable[[pd.Series], float]=np.sum,
- target_field: str='kNN_scores',
- partition_freq: Union[float, str]=np.inf,
- partition_min: int=2,
- kNN_algorithm: Literal["ball_tree", "kd_tree", "brute", "auto"]='ball_tree',
- metric: str='minkowski',
- p: int=2,
+ n_neighbors: int = 10,
+ trafo: Callable[[pd.Series], pd.Series] = lambda x: x,
+ trafo_on_partition: bool = True,
+ scoring_func: Callable[[pd.Series], float] = np.sum,
+ target_field: str = 'kNN_scores',
+ partition_freq: Union[float, str] = np.inf,
+ partition_min: int = 2,
+ kNN_algorithm: Literal["ball_tree", "kd_tree", "brute", "auto"] = 'ball_tree',
+ metric: str = 'minkowski',
+ p: int = 2,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
+ TODO: docstring need a rework
Score datapoints by an aggregation of the dictances to their k nearest neighbors.
The function is a wrapper around the NearestNeighbors method from pythons sklearn library (See reference [1]).
diff --git a/saqc/funcs/transformation.py b/saqc/funcs/transformation.py
index fdc99abbf..6a176b4a9 100644
--- a/saqc/funcs/transformation.py
+++ b/saqc/funcs/transformation.py
@@ -18,10 +18,9 @@ def transform(
field: str,
flagger: Flagger,
func: Callable[[pd.Series], pd.Series],
- partition_freq: Optional[Union[float, str]]=None,
+ partition_freq: Optional[Union[float, str]] = None,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
-
"""
Function to transform data columns with a transformation that maps series onto series of the same length.
@@ -75,5 +74,3 @@ def transform(
data[field] = val_ser
return data, flagger
-
-
diff --git a/saqc/lib/tools.py b/saqc/lib/tools.py
index dec366b98..6edfb0471 100644
--- a/saqc/lib/tools.py
+++ b/saqc/lib/tools.py
@@ -308,18 +308,6 @@ def isQuoted(string):
return bool(re.search(r"'.*'|\".*\"", string))
-# TODO: GL167
-def getDropMask(field, to_drop, flagger, default):
- drop_mask = pd.Series(False, index=flagger[field].index)
- if to_drop is None:
- to_drop = default
- to_drop = toSequence(to_drop)
- if len(to_drop) > 0:
- # drop_mask |= flagger.isFlagged(field, flag=to_drop)
- drop_mask |= flagger[field] == to_drop
- return drop_mask
-
-
def mutateIndex(index, old_name, new_name):
pos = index.get_loc(old_name)
index = index.drop(index[pos])
--
GitLab
From c549a6136404e79eebeaa29fb053b1000990e7d6 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 23 Mar 2021 19:54:53 +0100
Subject: [PATCH 076/180] made SaQC.modules.functions (class) signatures equal
to module/package-function signatures
---
saqc/core/modules/breaks.py | 28 ++++++++--
saqc/core/modules/changepoints.py | 51 +++++++++---------
saqc/core/modules/constants.py | 26 ++++++---
saqc/core/modules/curvefit.py | 22 ++++----
saqc/core/modules/drift.py | 84 +++++++++++++++--------------
saqc/core/modules/flagtools.py | 31 ++++++-----
saqc/core/modules/generic.py | 10 ++--
saqc/core/modules/interpolation.py | 65 ++++++----------------
saqc/core/modules/outliers.py | 80 ++++++++++++++-------------
saqc/core/modules/pattern.py | 21 ++++----
saqc/core/modules/resampling.py | 65 +++++++++++-----------
saqc/core/modules/residues.py | 20 +++----
saqc/core/modules/rolling.py | 12 ++---
saqc/core/modules/scores.py | 11 ++--
saqc/core/modules/tools.py | 38 +++++--------
saqc/core/modules/transformation.py | 8 +--
16 files changed, 295 insertions(+), 277 deletions(-)
diff --git a/saqc/core/modules/breaks.py b/saqc/core/modules/breaks.py
index 2a600aa85..02b237a60 100644
--- a/saqc/core/modules/breaks.py
+++ b/saqc/core/modules/breaks.py
@@ -1,18 +1,40 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
+from typing import Tuple
import numpy as np
+from dios import DictOfSeries
+from saqc import Flagger
from saqc.core.modules.base import ModuleBase
+from saqc.lib.types import FreqString, IntegerWindow, ColumnName
class Breaks(ModuleBase):
- def flagMissing(self, field: str, nodata=np.nan, **kwargs):
+ def flagMissing(
+ self,
+ field: ColumnName,
+ nodata: float = np.nan,
+ **kwargs
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagMissing", locals())
- def flagIsolated(self, field: str, gap_window: str, group_window: str, **kwargs):
+ def flagIsolated(
+ self,
+ field: ColumnName,
+ gap_window: FreqString,
+ group_window: FreqString,
+ **kwargs
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagIsolated", locals())
- def flagJumps(self, field: str, thresh: float, winsz: str, min_periods: int = 1, **kwargs):
+ def flagJumps(
+ self,
+ field: ColumnName,
+ thresh: float,
+ winsz: FreqString,
+ min_periods: IntegerWindow = 1,
+ **kwargs
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagJumps", locals())
diff --git a/saqc/core/modules/changepoints.py b/saqc/core/modules/changepoints.py
index 58093bc17..bab02fc86 100644
--- a/saqc/core/modules/changepoints.py
+++ b/saqc/core/modules/changepoints.py
@@ -1,49 +1,50 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from typing import Callable, Union
-from typing_extensions import Literal
+from typing import Callable, Optional, Tuple
+
import numpy as np
+from dios import DictOfSeries
+from typing_extensions import Literal
+
+from saqc import Flagger
from saqc.core.modules.base import ModuleBase
+from saqc.lib.types import FreqString, IntegerWindow
class ChangePoints(ModuleBase):
def flagChangePoints(
- self,
- field: str,
- stat_func: Callable[[np.array], np.array],
- thresh_func: Callable[[np.array], np.array],
- bwd_window: str,
- min_periods_bwd: Union[str, int],
- fwd_window: str = None,
- min_periods_fwd: Union[str, int] = None,
+ self, field: str,
+ stat_func: Callable[[np.ndarray, np.ndarray], float],
+ thresh_func: Callable[[np.ndarray, np.ndarray], float],
+ bwd_window: FreqString,
+ min_periods_bwd: IntegerWindow,
+ fwd_window: Optional[FreqString] = None,
+ min_periods_fwd: Optional[IntegerWindow] = None,
closed: Literal["right", "left", "both", "neither"] = "both",
- try_to_jit: bool = True,
- reduce_window: str = None,
- reduce_func: Callable[[np.array, np.array], np.array] = lambda x, y: x.argmax(),
+ try_to_jit: bool = True, # TODO rm, not a user decision
+ reduce_window: FreqString = None,
+ reduce_func: Callable[[np.ndarray, np.ndarray], int] = lambda x, _: x.argmax(),
**kwargs
- ):
-
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagChangePoints", locals())
def assignChangePointCluster(
- self,
- field: str,
- stat_func: Callable[[np.array], np.array],
- thresh_func: Callable[[np.array], np.array],
+ self, field: str,
+ stat_func: Callable[[np.array, np.array], float],
+ thresh_func: Callable[[np.array, np.array], float],
bwd_window: str,
- min_periods_bwd: Union[str, int],
+ min_periods_bwd: int,
fwd_window: str = None,
- min_periods_fwd: Union[str, int] = None,
+ min_periods_fwd: Optional[int] = None,
closed: Literal["right", "left", "both", "neither"] = "both",
- try_to_jit: bool = True,
+ try_to_jit: bool = True, # TODO: rm, not a user decision
reduce_window: str = None,
- reduce_func: Callable[[np.array, np.array], np.array] = lambda x, y: x.argmax(),
+ reduce_func: Callable[[np.ndarray, np.ndarray], float] = lambda x, _: x.argmax(),
model_by_resids: bool = False,
flag_changepoints: bool = False,
assign_cluster: bool = True,
**kwargs
- ):
-
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("assignChangePointCluster", locals())
diff --git a/saqc/core/modules/constants.py b/saqc/core/modules/constants.py
index e7d598bf4..09f55eb00 100644
--- a/saqc/core/modules/constants.py
+++ b/saqc/core/modules/constants.py
@@ -1,20 +1,32 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
+from typing import Tuple
+from dios import DictOfSeries
+
+from saqc import Flagger
from saqc.core.modules.base import ModuleBase
+from saqc.lib.types import FreqString, ColumnName
class Constants(ModuleBase):
def flagByVariance(
- self, field: str,
- window: str = "12h",
- thresh: float = 0.0005,
- max_missing: int = None,
- max_consec_missing: int = None,
+ self,
+ field: ColumnName,
+ window: FreqString="12h",
+ thresh: float=0.0005,
+ max_missing: int=None,
+ max_consec_missing: int=None,
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagByVariance", locals())
- def flagConstants(self, field: str, thresh: float, window: str, **kwargs):
+ def flagConstants(
+ self,
+ field: ColumnName,
+ thresh: float,
+ window: FreqString,
+ **kwargs
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagConstants", locals())
diff --git a/saqc/core/modules/curvefit.py b/saqc/core/modules/curvefit.py
index 3cbac0abb..595126406 100644
--- a/saqc/core/modules/curvefit.py
+++ b/saqc/core/modules/curvefit.py
@@ -1,21 +1,21 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from typing import Union
+from typing import Union, Tuple
+from dios import DictOfSeries
from typing_extensions import Literal
+from saqc import Flagger
from saqc.core.modules.base import ModuleBase
class Curvefit(ModuleBase):
- def fitPolynomial(
- self,
- field: str,
- winsz: Union[int, str],
- polydeg: int,
- numba: Literal[True, False, "auto"] = "auto",
- eval_flags: bool = True,
- min_periods: int = 0,
- return_residues: bool = False,
- **kwargs):
+ def fitPolynomial(self, field: str,
+ winsz: Union[int, str],
+ polydeg: int,
+ numba: Literal[True, False, "auto"] = "auto",
+ eval_flags: bool = True,
+ min_periods: int = 0,
+ return_residues: bool = False,
+ **kwargs) -> Tuple[DictOfSeries, Flagger]:
return self.defer("fitPolynomial", locals())
diff --git a/saqc/core/modules/drift.py b/saqc/core/modules/drift.py
index d6adbaeb5..3c422fc93 100644
--- a/saqc/core/modules/drift.py
+++ b/saqc/core/modules/drift.py
@@ -1,82 +1,84 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from typing import Sequence, Callable, Any, Optional
-from typing_extensions import Literal
+from typing import Sequence, Callable, Optional, Tuple
import numpy as np
from scipy.spatial.distance import pdist
from saqc.core.modules.base import ModuleBase
+from saqc.funcs import LinkageString, DictOfSeries, Flagger
+from saqc.lib.types import ColumnName, FreqString, CurveFitter
class Drift(ModuleBase):
def flagDriftFromNorm(
- self,
- field: str,
- fields: Sequence[str],
- segment_freq: str,
+ self,
+ field: ColumnName,
+ fields: Sequence[ColumnName],
+ segment_freq: FreqString,
norm_spread: float,
- norm_frac: float=0.5,
- metric: Callable[[np.array, np.array], float]=lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
- linkage_method: Literal["single", "complete", "average", "weighted", "centroid", "median", "ward"]="single",
+ norm_frac: float = 0.5,
+ metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
+ linkage_method: LinkageString = "single",
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagDriftFromNorm", locals())
def flagDriftFromReference(
- self,
- field: str,
- fields: Sequence[str],
- segment_freq: str,
+ self,
+ field: ColumnName,
+ fields: Sequence[ColumnName],
+ segment_freq: FreqString,
thresh: float,
- metric: Callable[[np.array, np.array], float]=lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
+ metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagDriftFromReference", locals())
def flagDriftFromScaledNorm(
- self,
- field: str,
- fields_scale1: Sequence[str],
- fields_scale2: Sequence[str],
- segment_freq: str,
+ self,
+ field: ColumnName,
+ fields_scale1: Sequence[ColumnName],
+ fields_scale2: Sequence[ColumnName],
+ segment_freq: FreqString,
norm_spread: float,
- norm_frac: float=0.5,
- metric: Callable[[np.array, np.array], float]=lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
- linkage_method: Literal["single", "complete", "average", "weighted", "centroid", "median", "ward"]="single",
+ norm_frac: float = 0.5,
+ metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
+ linkage_method: LinkageString = "single",
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagDriftFromScaledNorm", locals())
def correctExponentialDrift(
- self,
- field: str,
- maint_data_field: str,
+ self,
+ field: ColumnName,
+ maint_data_field: ColumnName,
cal_mean: int = 5,
flag_maint_period: bool = False,
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("correctExponentialDrift", locals())
def correctRegimeAnomaly(
- self,
- field: str,
- cluster_field: str,
- model: Callable[[np.array, Any], np.array],
- regime_transmission: Optional[str] = None,
- x_date: bool = False
- ):
+ self,
+ field: ColumnName,
+ cluster_field: ColumnName,
+ model: CurveFitter,
+ regime_transmission: Optional[FreqString] = None,
+ x_date: bool = False,
+ **kwargs
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("correctRegimeAnomaly", locals())
def correctOffset(
- self,
- field: str,
+ self,
+ field: ColumnName,
max_mean_jump: float,
normal_spread: float,
- search_winsz: str,
+ search_winsz: FreqString,
min_periods: int,
- regime_transmission: Optional[str] = None,
+ regime_transmission: Optional[FreqString] = None,
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("correctOffset", locals())
diff --git a/saqc/core/modules/flagtools.py b/saqc/core/modules/flagtools.py
index 637dc6f85..426dfb276 100644
--- a/saqc/core/modules/flagtools.py
+++ b/saqc/core/modules/flagtools.py
@@ -1,42 +1,47 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from typing import Any, Optional, Union
-from typing_extensions import Literal
+from typing import Any, Union, Tuple
import pandas as pd
-
from dios.dios import DictOfSeries
-from saqc.core.modules.base import ModuleBase
+from typing_extensions import Literal
+
+from saqc import Flagger
from saqc.constants import *
+from saqc.core.modules.base import ModuleBase
+from saqc.lib.types import ColumnName
class FlagTools(ModuleBase):
- def clearFlags(self, field: str, **kwargs):
+ def clearFlags(self, field: ColumnName, **kwargs) -> Tuple[DictOfSeries, Flagger]:
return self.defer("clearFlags", locals())
- def forceFlags(self, field: str, flag: float = BAD, **kwargs):
+ def forceFlags(
+ self, field: ColumnName, flag: float = BAD, **kwargs
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("forceFlags", locals())
- def flagDummy(self, field: str, **kwargs):
+ def flagDummy(self, field: ColumnName, **kwargs) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagDummy", locals())
- def flagForceFail(self, field: str, **kwargs):
+ def flagForceFail(self, field: ColumnName, **kwargs):
return self.defer("flagForceFail", locals())
- def flagUnflagged(self, field: str, flag: float = BAD, **kwargs):
+ def flagUnflagged(
+ self, field: ColumnName, flag: float = BAD, **kwargs
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagUnflagged", locals())
- def flagGood(self, field: str, flag: float = BAD, **kwargs):
+ def flagGood(self, field: ColumnName, flag=BAD, **kwargs) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagGood", locals())
def flagManual(
- self,
- field: str,
+ self, field: ColumnName,
mdata: Union[pd.Series, pd.DataFrame, DictOfSeries],
mflag: Any = 1,
method=Literal["plain", "ontime", "left-open", "right-open"],
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagManual", locals())
diff --git a/saqc/core/modules/generic.py b/saqc/core/modules/generic.py
index 43b792590..3f44c45f7 100644
--- a/saqc/core/modules/generic.py
+++ b/saqc/core/modules/generic.py
@@ -1,18 +1,22 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from typing import Callable
+from typing import Callable, Tuple
import numpy as np
import pandas as pd
+from dios import DictOfSeries
+from saqc import Flagger, BAD
from saqc.core.modules.base import ModuleBase
class Generic(ModuleBase):
- def process(self, field: str, func: Callable[[pd.Series], pd.Series], nodata: float = np.nan, **kwargs):
+ def process(self, field: str, func: Callable[[pd.Series], pd.Series],
+ nodata: float = np.nan, **kwargs) -> Tuple[DictOfSeries, Flagger]:
return self.defer("process", locals())
- def flag(self, field: str, func: Callable[[pd.Series], pd.Series], nodata: float = np.nan, **kwargs):
+ def flag(self, field: str, func: Callable[[pd.Series], pd.Series],
+ nodata: float = np.nan, flag=BAD, **kwargs) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flag", locals())
diff --git a/saqc/core/modules/interpolation.py b/saqc/core/modules/interpolation.py
index 42db500c2..c9aed6105 100644
--- a/saqc/core/modules/interpolation.py
+++ b/saqc/core/modules/interpolation.py
@@ -1,84 +1,53 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from typing import Union, Callable, Any, Optional, Sequence
-from typing_extensions import Literal
+from typing import Union, Callable, Any, Optional, Sequence, Tuple
import numpy as np
import pandas as pd
+from dios import DictOfSeries
+from typing_extensions import Literal
+from saqc import Flagger
from saqc.constants import *
from saqc.core.modules.base import ModuleBase
+from saqc.funcs.interpolation import _SUPPORTED_METHODS
class Interpolation(ModuleBase):
def interpolateByRolling(
- self,
- field: str,
+ self, field: str,
winsz: Union[str, int],
- func: Callable[[pd.Series], pd.Series] = np.median,
+ func: Callable[[pd.Series], float] = np.median,
center: bool = True,
min_periods: int = 0,
- interpol_flag=Any,
+ flag: float = UNFLAGGED,
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("interpolateByRolling", locals())
def interpolateInvalid(
- self,
+ self,
field: str,
- method: Literal["linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric", "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"],
+ method: _SUPPORTED_METHODS,
inter_order: int = 2,
inter_limit: int = 2,
- interpol_flag: float = UNFLAGGED,
downgrade_interpolation: bool = False,
- not_interpol_flags: Optional[Union[float, Sequence[float]]] = None,
+ flag: float = UNFLAGGED,
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("interpolateInvalid", locals())
def interpolateIndex(
- self,
+ self,
field: str,
freq: str,
- method: Literal["linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric", "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"],
- inter_order: int = 2,
- to_drop: Optional[Union[Any, Sequence[Any]]] = None,
- downgrade_interpolation: bool = False,
- empty_intervals_flag: Any = None,
- grid_field: str = None,
- inter_limit: int = 2,
- freq_check: Optional[Literal["check", "auto"]] = None,
- **kwargs
- ):
- return self.defer("interpolateIndex", locals())
-
- def interpolateInvalid(
- self,
- field: str,
- method: Literal["linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric", "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"],
+ method: _SUPPORTED_METHODS,
inter_order: int = 2,
inter_limit: int = 2,
- interpol_flag: float = UNFLAGGED,
downgrade_interpolation: bool = False,
- not_interpol_flags: Optional[Union[Any, Sequence[Any]]] = None,
**kwargs
- ):
- return self.defer("interpolateInvalid", locals())
-
- def interpolateIndex(
- self,
- field: str,
- freq: str,
- method: Literal["linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric", "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"],
- inter_order: int = 2,
- to_drop: Optional[Union[Any, Sequence[Any]]] = None,
- downgrade_interpolation: bool = False,
- empty_intervals_flag: Any = None,
- grid_field: str = None,
- inter_limit: int = 2,
- freq_check: Optional[Literal["check", "auto"]] = None,
- **kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("interpolateIndex", locals())
+
diff --git a/saqc/core/modules/outliers.py b/saqc/core/modules/outliers.py
index cb9d9849f..08efb1068 100644
--- a/saqc/core/modules/outliers.py
+++ b/saqc/core/modules/outliers.py
@@ -1,103 +1,109 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from typing import Optional, Union, Callable, Sequence
-from typing_extensions import Literal
+from typing import Optional, Union, Callable, Sequence, Tuple
import numpy as np
import pandas as pd
+from dios import DictOfSeries
+from typing_extensions import Literal
+from saqc import Flagger
from saqc.core.modules.base import ModuleBase
+from saqc.lib.types import IntegerWindow, FreqString, ColumnName
class Outliers(ModuleBase):
def flagByStray(
- self,
- field: str,
- partition_freq: Optional[Union[str, int]] = None,
+ self,
+ field: ColumnName,
+ partition_freq: Optional[Union[IntegerWindow, FreqString]] = None,
partition_min: int = 11,
iter_start: float = 0.5,
alpha: float = 0.05,
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagByStray", locals())
def flagMVScores(
- self,
- field: str,
- fields: Sequence[str],
+ self,
+ field: ColumnName,
+ fields: Sequence[ColumnName],
trafo: Callable[[pd.Series], pd.Series] = lambda x: x,
alpha: float = 0.05,
n_neighbors: int = 10,
scoring_func: Callable[[pd.Series], float] = np.sum,
iter_start: float = 0.5,
- stray_partition: Optional[Union[str, int]] = None,
+ stray_partition: Optional[Union[IntegerWindow, FreqString]] = None,
stray_partition_min: int = 11,
trafo_on_partition: bool = True,
- reduction_range: Optional[str] = None,
- reduction_drop_flagged: bool = False,
+ reduction_range: Optional[FreqString] = None,
+ reduction_drop_flagged: bool = False, # TODO: still a case ?
reduction_thresh: float = 3.5,
reduction_min_periods: int = 1,
**kwargs,
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagMVScores", locals())
def flagRaise(
- self,
- field: str,
+ self,
+ field: ColumnName,
thresh: float,
- raise_window: str,
- intended_freq: str,
- average_window: Optional[str] = None,
+ raise_window: FreqString,
+ intended_freq: FreqString,
+ average_window: Optional[FreqString] = None,
mean_raise_factor: float = 2.,
min_slope: Optional[float] = None,
min_slope_weight: float = 0.8,
- numba_boost: bool = True,
+ numba_boost: bool = True, # TODO: rm, not a user decision
**kwargs,
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagRaise", locals())
- def flagMAD(self, field: str, window: str, z: float = 3.5, **kwargs):
+ def flagMAD(
+ self, field: ColumnName, window: FreqString, z: float = 3.5, **kwargs
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagMAD", locals())
def flagOffset(
- self,
- field: str,
+ self,
+ field: ColumnName,
thresh: float,
tolerance: float,
- window: str,
- numba_kickin: int = 200000,
+ window: Union[IntegerWindow, FreqString],
+ rel_thresh: Optional[float] = None,
+ numba_kickin: int = 200000, # TODO: rm, not a user decision
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagOffset", locals())
def flagByGrubbs(
- self,
- field: str,
- winsz: Union[str, int],
+ self,
+ field: ColumnName,
+ winsz: Union[FreqString, IntegerWindow],
alpha: float = 0.05,
min_periods: int = 8,
check_lagged: bool = False,
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagByGrubbs", locals())
def flagRange(
- self,
- field: str,
+ self,
+ field: ColumnName,
min: float = -np.inf,
max: float = np.inf,
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagRange", locals())
def flagCrossStatistic(
- self,
- field: str,
- fields: Sequence[str],
+ self,
+ field: ColumnName,
+ fields: Sequence[ColumnName],
thresh: float,
cross_stat: Literal["modZscore", "Zscore"] = "modZscore",
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagCrossStatistic", locals())
diff --git a/saqc/core/modules/pattern.py b/saqc/core/modules/pattern.py
index 96e5828a0..06c9ab26c 100644
--- a/saqc/core/modules/pattern.py
+++ b/saqc/core/modules/pattern.py
@@ -1,29 +1,32 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from typing import Sequence
+from typing import Sequence, Tuple
+from dios import DictOfSeries
+
+from saqc import Flagger
from saqc.core.modules.base import ModuleBase
class Pattern(ModuleBase):
def flagPatternByDTW(
- self,
+ self,
field: str,
ref_field: str,
- widths: Sequence[int] = (1, 2, 4, 8),
- waveform: str = "mexh",
+ widths: Sequence[int]=(1, 2, 4, 8),
+ waveform: str="mexh",
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagPatternByDTW", locals())
def flagPatternByWavelet(
- self,
+ self,
field: str,
ref_field: str,
- max_distance: float = 0.03,
- normalize: bool = True,
+ max_distance: float=0.03,
+ normalize: bool=True,
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagPatternByWavelet", locals())
diff --git a/saqc/core/modules/resampling.py b/saqc/core/modules/resampling.py
index adc769aae..be4859bb5 100644
--- a/saqc/core/modules/resampling.py
+++ b/saqc/core/modules/resampling.py
@@ -1,72 +1,73 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from typing import Optional, Union, Any, Sequence, Callable
-from typing_extensions import Literal
+from typing import Optional, Callable, Tuple
import numpy as np
import pandas as pd
+from dios import DictOfSeries
+from typing_extensions import Literal
+from saqc import Flagger
from saqc.core.modules.base import ModuleBase
+from saqc.funcs.interpolation import _SUPPORTED_METHODS
class Resampling(ModuleBase):
def aggregate(
- self,
+ self,
field: str,
freq: str,
value_func,
- flag_func: Callable[[pd.Series], pd.Series] = np.nanmax,
+ flag_func: Callable[[pd.Series], float] = np.nanmax,
method: Literal["fagg", "bagg", "nagg"] = "nagg",
- to_drop: Optional[Union[Any, Sequence[Any]]] = None,
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("aggregate", locals())
def linear(
- self,
+ self,
field: str,
freq: str,
- to_drop: Optional[Union[Any, Sequence[Any]]] = None,
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("linear", locals())
def interpolate(
- self,
+ self,
field: str,
freq: str,
- method: Literal["linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric", "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"],
+ method: _SUPPORTED_METHODS,
order: int = 1,
- to_drop: Optional[Union[Any, Sequence[Any]]] = None,
**kwargs,
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("interpolate", locals())
def mapToOriginal(
- self,
+ self,
field: str,
- method: Literal["inverse_fagg", "inverse_bagg", "inverse_nagg", "inverse_fshift", "inverse_bshift", "inverse_nshift", "inverse_interpolation"],
- to_drop: Optional[Union[Any, Sequence[Any]]] = None,
+ method: Literal[
+ "inverse_fagg", "inverse_bagg", "inverse_nagg",
+ "inverse_fshift", "inverse_bshift", "inverse_nshift",
+ "inverse_interpolation"
+ ],
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("mapToOriginal", locals())
def shift(
- self,
+ self,
field: str,
freq: str,
method: Literal["fshift", "bshift", "nshift"] = "nshift",
- to_drop: Optional[Union[Any, Sequence[Any]]] = None,
- empty_intervals_flag: Optional[str] = None,
- freq_check: Optional[Literal["check", "auto"]] = None,
+ freq_check: Optional[Literal["check", "auto"]] = None, # TODO: not a user decision
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("shift", locals())
def resample(
- self,
+ self,
field: str,
freq: str,
agg_func: Callable[[pd.Series], pd.Series] = np.mean,
@@ -75,23 +76,21 @@ class Resampling(ModuleBase):
max_invalid_consec_d: Optional[int] = None,
max_invalid_consec_f: Optional[int] = None,
max_invalid_total_f: Optional[int] = None,
- flag_agg_func: Callable[[pd.Series], pd.Series] = max,
- empty_intervals_flag: Optional[Any] = None,
- to_drop: Optional[Union[Any, Sequence[Any]]] = None,
- all_na_2_empty: bool = False,
+ flag_agg_func: Callable[[pd.Series], float] = max,
freq_check: Optional[Literal["check", "auto"]] = None,
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("resample", locals())
def reindexFlags(
- self,
+ self,
field: str,
- method: Literal["inverse_fagg", "inverse_bagg", "inverse_nagg", "inverse_fshift", "inverse_bshift", "inverse_nshift"],
+ method: Literal[
+ "inverse_fagg", "inverse_bagg", "inverse_nagg",
+ "inverse_fshift", "inverse_bshift", "inverse_nshift"
+ ],
source: str,
freq: Optional[str] = None,
- to_drop: Optional[Union[Any, Sequence[Any]]] = None,
- freq_check: Optional[Literal["check", "auto"]] = None,
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("reindexFlags", locals())
diff --git a/saqc/core/modules/residues.py b/saqc/core/modules/residues.py
index f3dd502e4..85d7426f0 100644
--- a/saqc/core/modules/residues.py
+++ b/saqc/core/modules/residues.py
@@ -1,36 +1,38 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from typing import Optional, Union, Callable
-from typing_extensions import Literal
+from typing import Optional, Union, Callable, Tuple
import numpy as np
+from dios import DictOfSeries
+from typing_extensions import Literal
+from saqc import Flagger
from saqc.core.modules.base import ModuleBase
class Residues(ModuleBase):
def calculatePolynomialResidues(
- self,
+ self,
field: str,
winsz: Union[str, int],
polydeg: int,
- numba: Literal[True, False, "auto"] = "auto",
- eval_flags: bool = True,
+ numba: Literal[True, False, "auto"] = "auto", # TODO: rm, not a a user decision
+ eval_flags: bool = True, # TODO, not valid anymore, if still needed, maybe assign user-passed ``flag``?
min_periods: Optional[int] = 0,
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("calculatePolynomialResidues", locals())
def calculateRollingResidues(
- self,
+ self,
field: str,
winsz: Union[str, int],
- func: Callable[[np.array], np.array] = np.mean,
+ func: Callable[[np.ndarray], np.ndarray] = np.mean,
eval_flags: bool = True,
min_periods: Optional[int] = 0,
center: bool = True,
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("calculateRollingResidues", locals())
diff --git a/saqc/core/modules/rolling.py b/saqc/core/modules/rolling.py
index a249fb470..f9c6be163 100644
--- a/saqc/core/modules/rolling.py
+++ b/saqc/core/modules/rolling.py
@@ -11,14 +11,14 @@ from saqc.core.modules.base import ModuleBase
class Rolling(ModuleBase):
def roll(
- self,
+ self,
field: str,
winsz: Union[str, int],
- func: Callable[[pd.Series], float] = np.mean,
- eval_flags: bool = True,
- min_periods: int = 0,
- center: bool = True,
- return_residues=False,
+ func: Callable[[pd.Series], float]=np.mean,
+ eval_flags: bool=True, # TODO: not applicable anymore
+ min_periods: int=0,
+ center: bool=True,
+ return_residues=False, # TODO: this should not be public, a wrapper would be better
**kwargs
):
return self.defer("roll", locals())
diff --git a/saqc/core/modules/scores.py b/saqc/core/modules/scores.py
index 8a25f3e09..a36e73d88 100644
--- a/saqc/core/modules/scores.py
+++ b/saqc/core/modules/scores.py
@@ -1,19 +1,21 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from typing import Sequence, Callable, Union, Optional
-from typing_extensions import Literal
+from typing import Sequence, Callable, Union, Tuple
import numpy as np
import pandas as pd
+from dios import DictOfSeries
+from typing_extensions import Literal
+from saqc import Flagger
from saqc.core.modules.base import ModuleBase
class Scores(ModuleBase):
def assignKNNScore(
- self,
+ self,
field: str,
fields: Sequence[str],
n_neighbors: int = 10,
@@ -26,7 +28,6 @@ class Scores(ModuleBase):
kNN_algorithm: Literal["ball_tree", "kd_tree", "brute", "auto"] = 'ball_tree',
metric: str = 'minkowski',
p: int = 2,
- radius: Optional[float] = None,
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("assignKNNScore", locals())
diff --git a/saqc/core/modules/tools.py b/saqc/core/modules/tools.py
index 5260a8fb3..7d3c39859 100644
--- a/saqc/core/modules/tools.py
+++ b/saqc/core/modules/tools.py
@@ -1,43 +1,33 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from typing import Optional
+from typing import Optional, Tuple
+
+from dios import DictOfSeries
from typing_extensions import Literal
+from saqc import Flagger
from saqc.core.modules.base import ModuleBase
class Tools(ModuleBase):
- def copy(
- self,
- field: str,
- new_field: str,
- **kwargs
- ):
+ def copy(self, field: str, new_field: str, **kwargs) -> Tuple[DictOfSeries, Flagger]:
return self.defer("copy", locals())
- def drop(
- self,
- field: str,
- **kwargs
- ):
+ def drop(self, field: str, **kwargs) -> Tuple[DictOfSeries, Flagger]:
return self.defer("drop", locals())
- def rename(
- self,
- field: str,
- new_name: str,
- **kwargs
- ):
+ def rename(self, field: str, new_name: str, **kwargs) -> Tuple[DictOfSeries, Flagger]:
return self.defer("rename", locals())
def mask(
- self,
+ self,
field: str,
mode: Literal["periodic", "mask_var"],
- mask_var: Optional[str] = None,
- period_start: Optional[str] = None,
- period_end: Optional[str] = None,
- include_bounds: bool = True
- ):
+ mask_var: Optional[str]=None,
+ period_start: Optional[str]=None,
+ period_end: Optional[str]=None,
+ include_bounds: bool=True,
+ **kwargs,
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("mask", locals())
diff --git a/saqc/core/modules/transformation.py b/saqc/core/modules/transformation.py
index 7b5755d44..f85053fc5 100644
--- a/saqc/core/modules/transformation.py
+++ b/saqc/core/modules/transformation.py
@@ -1,20 +1,22 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from typing import Callable, Optional, Union
+from typing import Callable, Optional, Union, Tuple
import pandas as pd
+from dios import DictOfSeries
+from saqc import Flagger
from saqc.core.modules.base import ModuleBase
class Transformation(ModuleBase):
def transform(
- self,
+ self,
field: str,
func: Callable[[pd.Series], pd.Series],
partition_freq: Optional[Union[float, str]] = None,
**kwargs
- ):
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("transform", locals())
--
GitLab
From e44e2db7c6c813054354a000a57dac6e02e2de1e Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 23 Mar 2021 20:00:41 +0100
Subject: [PATCH 077/180] added signatureMaker, a code generation script
---
signatureMaker.py | 159 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 159 insertions(+)
create mode 100644 signatureMaker.py
diff --git a/signatureMaker.py b/signatureMaker.py
new file mode 100644
index 000000000..c68ed4b93
--- /dev/null
+++ b/signatureMaker.py
@@ -0,0 +1,159 @@
+from saqc.funcs import *
+from saqc.core.register import FUNC_MAP
+import os
+
+
+def start_with_exactly_N_spaces(line: str, N: int):
+ return line.startswith(' ' * N) and not line.startswith(' ' * (N + 1))
+
+
+def find_original_signature(fh, fname):
+ sig = []
+ start = end = False
+ for line in fh.readlines():
+
+ # find start of signature
+ if not start:
+
+ if line.startswith(f'def {fname}'):
+ sig.append(line)
+ start = True
+ continue
+
+ # find end of signature
+ if '"""' in line or start_with_exactly_N_spaces(line, 4):
+ end = True
+ break # do not append line
+
+ # found last line of signature
+ if '->' in line:
+ end = True
+
+ sig.append(line)
+
+ if end:
+ break
+
+ # if end or/and start was not found,
+ # something went wrong
+ if end is False:
+ sig = None
+
+ return sig
+
+
+def replace_core_signatures(readlines, writelines, fname, fh):
+ start = end = False
+ for line in readlines:
+
+ # append the rest of the file, the loop ends here
+ if end is True:
+ fh.write(line)
+ continue
+
+ # find start of signature, loop starts here
+ if not start:
+
+ if line.startswith(f' def {fname}'):
+ start = True
+
+ # insert the replacement
+ for rline in writelines:
+ fh.write(' ')
+ fh.write(rline)
+
+ else:
+ fh.write(line)
+ continue
+
+ # found line after end of signature
+ if '"""' in line or start_with_exactly_N_spaces(line, 8):
+ end = True
+ fh.write(line)
+ continue
+
+ # found last line of signature
+ if '->' in line:
+ end = True
+ continue
+
+
+def replace_datafieldflagger(lines):
+ import re
+ empty = re.compile(' *\n')
+ data = re.compile('.*(data[=: ][^,]*, ?)') # eg. 'data: DictOfSeries,'
+ flagger = re.compile('.*(flagger[=: ][^,]*, ?)') # eg. 'flagger: Flagger,'
+ pattern = [data, flagger]
+ i = 0
+ replaced = []
+ for line in lines:
+ if 'copy' in line:
+ i = i
+
+ if i < len(pattern):
+ found = pattern[i].match(line)
+ if found:
+ match = found[1] # regex group
+ replacement = ''
+ if i == 0:
+ replacement = 'self, '
+ line = line.replace(match, replacement, 1)
+ i += 1
+
+ # find in same line
+ for j in range(i, len(pattern)):
+ found = pattern[i].match(line)
+ if found:
+ line = line.replace(found[1], '', 1)
+ i += 1
+
+ empty_line = empty.match(line)
+ if empty_line:
+ continue
+
+ replaced.append(line)
+
+ return replaced
+
+
+def autoreplace_signature():
+ postfix = '_autosignature'
+ saqc_path = 'saqc/core/modules/'
+ touched_modules = []
+
+ for name in FUNC_MAP:
+ module, fname = name.split('.')
+
+ with open(f'saqc/funcs/{module}.py', 'r') as fh:
+ lines = find_original_signature(fh, fname)
+
+ if lines is None:
+ warnings.warn(f"end of signature of '{fname}' not found - ignoring")
+ continue
+
+ lines = replace_datafieldflagger(lines)
+ print(''.join(lines))
+
+ readfile = f'{saqc_path}{module}.py'
+ writefile = f'{saqc_path}{module}{postfix}.py'
+ if module in touched_modules:
+ readfile = writefile
+ else:
+ touched_modules.append(module)
+
+ with open(readfile, 'r') as fh:
+ readlines = fh.readlines()
+
+ with open(writefile, 'w') as fh:
+ replace_core_signatures(readlines, lines, fname, fh)
+
+ files = os.listdir('saqc/core/modules/')
+ for new in files:
+ if postfix in new:
+ old = saqc_path + new.replace(postfix, '')
+ new = saqc_path + new
+ os.replace(new, old)
+
+
+if __name__ == '__main__':
+ autoreplace_signature()
--
GitLab
From 438f68b8983ed3adf77b12a882290f61ee2b4d5a Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 23 Mar 2021 20:46:05 +0100
Subject: [PATCH 078/180] improved signatureMaker to be more readable
---
signatureMaker.py | 118 ++++++++++++++++++++++++++++++++--------------
1 file changed, 83 insertions(+), 35 deletions(-)
diff --git a/signatureMaker.py b/signatureMaker.py
index c68ed4b93..f31e9303f 100644
--- a/signatureMaker.py
+++ b/signatureMaker.py
@@ -1,3 +1,6 @@
+import re
+from typing import List
+
from saqc.funcs import *
from saqc.core.register import FUNC_MAP
import os
@@ -7,7 +10,24 @@ def start_with_exactly_N_spaces(line: str, N: int):
return line.startswith(' ' * N) and not line.startswith(' ' * (N + 1))
-def find_original_signature(fh, fname):
+def find_original_signature(fh, func_name):
+ """
+ Extract the signature code from a file.
+
+ Parameters
+ ----------
+ fh : file
+ file descriptor, to read the code from
+
+ func_name : str
+ function name, of the signature in question
+
+ Returns
+ -------
+ lines: list or None
+ list of lines of code if found, otherwise None.
+
+ """
sig = []
start = end = False
for line in fh.readlines():
@@ -15,7 +35,7 @@ def find_original_signature(fh, fname):
# find start of signature
if not start:
- if line.startswith(f'def {fname}'):
+ if line.startswith(f'def {func_name}'):
sig.append(line)
start = True
continue
@@ -42,34 +62,52 @@ def find_original_signature(fh, fname):
return sig
-def replace_core_signatures(readlines, writelines, fname, fh):
+def replace_core_signatures(core_code: List[str], sig_code: List[str], func_name: str, target_file):
+ """
+ Replace a signature in the core code with a signature from a module.
+
+ Parameters
+ ----------
+ core_code : list
+ lines of code, one by one
+
+ sig_code : list
+ lines of code, one by one (only the signature in question)
+
+ func_name : str
+ function name in question
+
+ target_file : file
+ file descriptor to write the modified code to
+ """
start = end = False
- for line in readlines:
+ for line in core_code:
# append the rest of the file, the loop ends here
if end is True:
- fh.write(line)
+ target_file.write(line)
continue
# find start of signature, loop starts here
if not start:
- if line.startswith(f' def {fname}'):
+ if line.startswith(f' def {func_name}'):
start = True
# insert the replacement
- for rline in writelines:
- fh.write(' ')
- fh.write(rline)
+ for rline in sig_code:
+ target_file.write(' ')
+ target_file.write(rline)
+ # start of sig, not found yet
else:
- fh.write(line)
+ target_file.write(line)
continue
# found line after end of signature
if '"""' in line or start_with_exactly_N_spaces(line, 8):
end = True
- fh.write(line)
+ target_file.write(line)
continue
# found last line of signature
@@ -79,33 +117,31 @@ def replace_core_signatures(readlines, writelines, fname, fh):
def replace_datafieldflagger(lines):
- import re
+ """
+ Remove 'data' and 'flagger' from signature, and insert 'self' instead.
+ """
empty = re.compile(' *\n')
data = re.compile('.*(data[=: ][^,]*, ?)') # eg. 'data: DictOfSeries,'
flagger = re.compile('.*(flagger[=: ][^,]*, ?)') # eg. 'flagger: Flagger,'
- pattern = [data, flagger]
+ pattern_list = [data, flagger]
i = 0
replaced = []
+
for line in lines:
- if 'copy' in line:
- i = i
-
- if i < len(pattern):
- found = pattern[i].match(line)
- if found:
- match = found[1] # regex group
- replacement = ''
- if i == 0:
- replacement = 'self, '
- line = line.replace(match, replacement, 1)
- i += 1
-
- # find in same line
- for j in range(i, len(pattern)):
- found = pattern[i].match(line)
- if found:
- line = line.replace(found[1], '', 1)
- i += 1
+
+ if i < len(pattern_list):
+
+ # search for one patter after the other in the current line,
+ # if any is NOT found, we stop and continued from there in the
+ # next line (next loop outer integration)
+ for j in range(i, len(pattern_list)):
+ found = pattern_list[i].match(line)
+ if found:
+ # we replace the first match ('data') with 'self'
+ line = line.replace(found[1], 'self, ' if i == 0 else '', 1)
+ i += 1 # next pattern please
+ else:
+ break
empty_line = empty.match(line)
if empty_line:
@@ -116,11 +152,16 @@ def replace_datafieldflagger(lines):
return replaced
-def autoreplace_signature():
+def autoreplace_signatures():
+ """
+ Replaces core-signatures with the module-signatures, one-by-one.
+ """
postfix = '_autosignature'
saqc_path = 'saqc/core/modules/'
touched_modules = []
+ # one-by-one: we only process one signature at a time, this means
+ # that we see most files multiple times.
for name in FUNC_MAP:
module, fname = name.split('.')
@@ -131,9 +172,13 @@ def autoreplace_signature():
warnings.warn(f"end of signature of '{fname}' not found - ignoring")
continue
+ # modify original function signature
lines = replace_datafieldflagger(lines)
print(''.join(lines))
+ # find the right file. If we already processed a signature
+ # of the same module, we already have a modified file, so we
+ # need to read and write the same.
readfile = f'{saqc_path}{module}.py'
writefile = f'{saqc_path}{module}{postfix}.py'
if module in touched_modules:
@@ -141,12 +186,16 @@ def autoreplace_signature():
else:
touched_modules.append(module)
+ # READ
with open(readfile, 'r') as fh:
readlines = fh.readlines()
+ # WRITE
+ # replace sig's in Saqc.module to a temporary file
with open(writefile, 'w') as fh:
replace_core_signatures(readlines, lines, fname, fh)
+ # replace all original files with the temporary ones
files = os.listdir('saqc/core/modules/')
for new in files:
if postfix in new:
@@ -154,6 +203,5 @@ def autoreplace_signature():
new = saqc_path + new
os.replace(new, old)
-
if __name__ == '__main__':
- autoreplace_signature()
+ autoreplace_signatures()
--
GitLab
From 4a46ac152e3f0fb119d3f40dff788c0f66dd46dd Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 23 Mar 2021 23:38:57 +0100
Subject: [PATCH 079/180] rm docs/
---
docs/func_modules/breaks.py | 73 -----
docs/func_modules/changepoints.py | 107 --------
docs/func_modules/constants.py | 53 ----
docs/func_modules/curvefit.py | 67 -----
docs/func_modules/drift.py | 362 -------------------------
docs/func_modules/flagtools.py | 135 ---------
docs/func_modules/generic.py | 108 --------
docs/func_modules/interpolation.py | 123 ---------
docs/func_modules/module_dict.pkl | Bin 2466 -> 0 bytes
docs/func_modules/outliers.py | 359 ------------------------
docs/func_modules/pattern.py | 38 ---
docs/func_modules/resampling.py | 304 ---------------------
docs/func_modules/residues.py | 65 -----
docs/func_modules/rolling.py | 36 ---
docs/func_modules/scores.py | 81 ------
docs/func_modules/tools.py | 128 ---------
docs/func_modules/transformation.py | 25 --
docs/intro_modules/AdvancedFlagging.py | 66 -----
docs/intro_modules/BasicFlagging.py | 32 ---
docs/intro_modules/module_dict.pkl | Bin 238 -> 0 bytes
20 files changed, 2162 deletions(-)
delete mode 100644 docs/func_modules/breaks.py
delete mode 100644 docs/func_modules/changepoints.py
delete mode 100644 docs/func_modules/constants.py
delete mode 100644 docs/func_modules/curvefit.py
delete mode 100644 docs/func_modules/drift.py
delete mode 100644 docs/func_modules/flagtools.py
delete mode 100644 docs/func_modules/generic.py
delete mode 100644 docs/func_modules/interpolation.py
delete mode 100644 docs/func_modules/module_dict.pkl
delete mode 100644 docs/func_modules/outliers.py
delete mode 100644 docs/func_modules/pattern.py
delete mode 100644 docs/func_modules/resampling.py
delete mode 100644 docs/func_modules/residues.py
delete mode 100644 docs/func_modules/rolling.py
delete mode 100644 docs/func_modules/scores.py
delete mode 100644 docs/func_modules/tools.py
delete mode 100644 docs/func_modules/transformation.py
delete mode 100644 docs/intro_modules/AdvancedFlagging.py
delete mode 100644 docs/intro_modules/BasicFlagging.py
delete mode 100644 docs/intro_modules/module_dict.pkl
diff --git a/docs/func_modules/breaks.py b/docs/func_modules/breaks.py
deleted file mode 100644
index 115f1252e..000000000
--- a/docs/func_modules/breaks.py
+++ /dev/null
@@ -1,73 +0,0 @@
-"""
-Detecting breakish changes in timeseries value courses.
-
-This module provides functions to detect and flag breakish changes in the data value course, like gaps
-(:py:func:`flagMissing`), jumps/drops (:py:func:`flagJumps`) or isolated values (:py:func:`flagIsolated`).
-"""
-def flagMissing(field, nodata):
- """
- The function flags all values indicating missing data.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged.
- nodata : any, default np.nan
- A value that defines missing data.
- """
- pass
-
-
-def flagIsolated(field, gap_window, group_window):
- """
- The function flags arbitrary large groups of values, if they are surrounded by sufficiently
- large data gaps.
-
- A gap is a timespan containing either no data or invalid (usually `nan`) and flagged data only.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged.
- gap_window : str
- The minimum size of the gap before and after a group of valid values, making this group considered an
- isolated group. See condition (2) and (3)
- group_window : str
- The maximum temporal extension allowed for a group that is isolated by gaps of size 'gap_window',
- to be actually flagged as isolated group. See condition (1).
-
- Notes
- -----
- A series of values :math:`x_k,x_{k+1},...,x_{k+n}`, with associated timestamps :math:`t_k,t_{k+1},...,t_{k+n}`,
- is considered to be isolated, if:
-
- 1. :math:`t_{k+1} - t_n <` `group_window`
- 2. None of the :math:`x_j` with :math:`0 < t_k - t_j <` `gap_window`, is valid or unflagged (preceeding gap).
- 3. None of the :math:`x_j` with :math:`0 < t_j - t_(k+n) <` `gap_window`, is valid or unflagged (succeding gap).
-
- See Also
- --------
- :py:func:`flagMissing`
- """
- pass
-
-
-def flagJumps(field, thresh, winsz, min_periods):
- """
- Flag datapoints, where the mean of the values significantly changes (where the value course "jumps").
-
- Parameters
- ----------
- field : str
- The reference variable, the deviation from wich determines the flagging.
- thresh : float
- The threshold, the mean of the values have to change by, to trigger flagging.
- winsz : str
- The temporal extension, of the rolling windows, the mean values that are to be compared,
- are obtained from.
- min_periods : int, default 1
- Minimum number of periods that have to be present in a window of size `winsz`, so that
- the mean value obtained from that window is regarded valid.
- """
- pass
-
diff --git a/docs/func_modules/changepoints.py b/docs/func_modules/changepoints.py
deleted file mode 100644
index e2513f058..000000000
--- a/docs/func_modules/changepoints.py
+++ /dev/null
@@ -1,107 +0,0 @@
-"""
-
-"""
-def flagChangePoints(field, stat_func, thresh_func, bwd_window, min_periods_bwd, fwd_window, min_periods_fwd, closed, reduce_window, reduce_func):
- """
- Flag datapoints, where the parametrization of the process, the data is assumed to generate by, significantly
- changes.
-
- The change points detection is based on a sliding window search.
-
- Parameters
- ----------
- field : str
- The reference variable, the deviation from wich determines the flagging.
- stat_func : Callable[numpy.array, numpy.array]
- A function that assigns a value to every twin window. Left window content will be passed to first variable,
- right window content will be passed to the second.
- thresh_func : Callable[numpy.array, numpy.array]
- A function that determines the value level, exceeding wich qualifies a timestamps stat func value as denoting a
- changepoint.
- bwd_window : str
- The left (backwards facing) windows temporal extension (freq-string).
- min_periods_bwd : {str, int}
- Minimum number of periods that have to be present in a backwards facing window, for a changepoint test to be
- performed.
- fwd_window : {None, str}, default None
- The right (forward facing) windows temporal extension (freq-string).
- min_periods_fwd : {None, str, int}, default None
- Minimum number of periods that have to be present in a forward facing window, for a changepoint test to be
- performed.
- closed : {'right', 'left', 'both', 'neither'}, default 'both'
- Determines the closure of the sliding windows.
- reduce_window : {None, str}, default None
- The sliding window search method is not an exact CP search method and usually there wont be
- detected a single changepoint, but a "region" of change around a changepoint.
- If `reduce_window` is given, for every window of size `reduce_window`, there
- will be selected the value with index `reduce_func(x, y)` and the others will be dropped.
- If `reduce_window` is None, the reduction window size equals the
- twin window size, the changepoints have been detected with.
- reduce_func : Callable[[numpy.array, numpy.array], np.array], default lambda x, y: x.argmax()
- A function that must return an index value upon input of two arrays x and y.
- First input parameter will hold the result from the stat_func evaluation for every
- reduction window. Second input parameter holds the result from the thresh_func evaluation.
- The default reduction function just selects the value that maximizes the stat_func.
-
-
- Returns
- -------
- """
- pass
-
-
-def assignChangePointCluster(field, stat_func, thresh_func, bwd_window, min_periods_bwd, fwd_window, min_periods_fwd, closed, reduce_window, reduce_func, flag_changepoints, model_by_resids, assign_cluster):
- """
- Assigns label to the data, aiming to reflect continous regimes of the processes the data is assumed to be
- generated by.
- The regime change points detection is based on a sliding window search.
-
- Note, that the cluster labels will be stored to the `field` field of the input data, so that the data that is
- clustered gets overridden.
-
- Parameters
- ----------
- field : str
- The reference variable, the deviation from wich determines the flagging.
- stat_func : Callable[[numpy.array, numpy.array], float]
- A function that assigns a value to every twin window. Left window content will be passed to first variable,
- right window content will be passed to the second.
- thresh_func : Callable[numpy.array, numpy.array], float]
- A function that determines the value level, exceeding wich qualifies a timestamps stat func value as denoting a
- changepoint.
- bwd_window : str
- The left (backwards facing) windows temporal extension (freq-string).
- min_periods_bwd : {str, int}
- Minimum number of periods that have to be present in a backwards facing window, for a changepoint test to be
- performed.
- fwd_window : {None, str}, default None
- The right (forward facing) windows temporal extension (freq-string).
- min_periods_fwd : {None, str, int}, default None
- Minimum number of periods that have to be present in a forward facing window, for a changepoint test to be
- performed.
- closed : {'right', 'left', 'both', 'neither'}, default 'both'
- Determines the closure of the sliding windows.
- reduce_window : {None, str}, default None
- The sliding window search method is not an exact CP search method and usually there wont be
- detected a single changepoint, but a "region" of change around a changepoint.
- If `reduce_window` is given, for every window of size `reduce_window`, there
- will be selected the value with index `reduce_func(x, y)` and the others will be dropped.
- If `reduce_window` is None, the reduction window size equals the
- twin window size, the changepoints have been detected with.
- reduce_func : Callable[[numpy.array, numpy.array], numpy.array], default lambda x, y: x.argmax()
- A function that must return an index value upon input of two arrays x and y.
- First input parameter will hold the result from the stat_func evaluation for every
- reduction window. Second input parameter holds the result from the thresh_func evaluation.
- The default reduction function just selects the value that maximizes the stat_func.
- flag_changepoints : bool, default False
- If true, the points, where there is a change in data modelling regime detected get flagged bad.
- model_by_resids : bool, default False
- If True, the data is replaced by the stat_funcs results instead of regime labels.
- assign_cluster : bool, default True
- Is set to False, if called by function that oly wants to calculate flags.
-
- Returns
- -------
- """
- pass
-
diff --git a/docs/func_modules/constants.py b/docs/func_modules/constants.py
deleted file mode 100644
index dcb4f796a..000000000
--- a/docs/func_modules/constants.py
+++ /dev/null
@@ -1,53 +0,0 @@
-"""
-
-"""
-def flagConstants(field, thresh, window):
- """
- This functions flags plateaus/series of constant values of length `window` if
- their maximum total change is smaller than thresh.
-
- Function flags plateaus/series of constant values. Any interval of values y(t),..y(t+n) is flagged, if:
-
- (1) n > `window`
- (2) |(y(t + i) - (t + j)| < `thresh`, for all i,j in [0, 1, ..., n]
-
- Flag values are (semi-)constant.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged.
- thresh : float
- Upper bound for the maximum total change of an interval to be flagged constant.
- window : str
- Lower bound for the size of an interval to be flagged constant.
- """
- pass
-
-
-def flagByVariance(field, window, thresh, max_missing, max_consec_missing):
- """
- Function flags plateaus/series of constant values. Any interval of values y(t),..y(t+n) is flagged, if:
-
- (1) n > `window`
- (2) variance(y(t),...,y(t+n) < `thresh`
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged.
- window : str
- Only intervals of minimum size "window" have the chance to get flagged as constant intervals
- thresh : float
- The upper bound, the variance of an interval must not exceed, if the interval wants to be flagged a plateau.
- max_missing : {None, int}, default None
- Maximum number of nan values tolerated in an interval, for retrieving a valid
- variance from it. (Intervals with a number of nans exceeding "max_missing"
- have no chance to get flagged a plateau!)
- max_consec_missing : {None, int}, default None
- Maximum number of consecutive nan values allowed in an interval to retrieve a
- valid variance from it. (Intervals with a number of nans exceeding
- "max_consec_missing" have no chance to get flagged a plateau!)
- """
- pass
-
diff --git a/docs/func_modules/curvefit.py b/docs/func_modules/curvefit.py
deleted file mode 100644
index 4b67b225f..000000000
--- a/docs/func_modules/curvefit.py
+++ /dev/null
@@ -1,67 +0,0 @@
-"""
-
-"""
-def fitPolynomial(field, winsz, polydeg, numba, eval_flags, min_periods, return_residues):
- """
- Function fits a polynomial model to the data and returns the fitted data curve.
-
- The fit is calculated by fitting a polynomial of degree `polydeg` to a data slice
- of size `winsz`, that has x at its center.
-
- Note, that the resulting fit is stored to the `field` field of the input data, so that the original data, the
- polynomial is fitted to, gets overridden.
-
- Note, that, if data[field] is not alligned to an equidistant frequency grid, the window size passed,
- has to be an offset string. Also numba boost options don`t apply for irregularly sampled
- timeseries.
-
- Note, that calculating the residues tends to be quite costy, because a function fitting is perfomed for every
- sample. To improve performance, consider the following possibillities:
-
- In case your data is sampled at an equidistant frequency grid:
-
- (1) If you know your data to have no significant number of missing values, or if you do not want to
- calculate residues for windows containing missing values any way, performance can be increased by setting
- min_periods=winsz.
-
- (2) If your data consists of more then around 200000 samples, setting numba=True, will boost the
- calculations up to a factor of 5 (for samplesize > 300000) - however for lower sample sizes,
- numba will slow down the calculations, also, up to a factor of 5, for sample_size < 50000.
- By default (numba='auto'), numba is set to true, if the data sample size exceeds 200000.
-
- in case your data is not sampled at an equidistant frequency grid:
-
- (1) Harmonization/resampling of your data will have a noticable impact on polyfittings performance - since
- numba_boost doesnt apply for irregularly sampled data in the current implementation.
-
- Note, that in the current implementation, the initial and final winsz/2 values do not get fitted.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-modelled.
- winsz : {str, int}
- The size of the window you want to use for fitting. If an integer is passed, the size
- refers to the number of periods for every fitting window. If an offset string is passed,
- the size refers to the total temporal extension. The window will be centered around the vaule-to-be-fitted.
- For regularly sampled timeseries the period number will be casted down to an odd number if
- even.
- polydeg : int
- The degree of the polynomial used for fitting
- numba : {True, False, "auto"}, default "auto"
- Wheather or not to apply numbas just-in-time compilation onto the poly fit function. This will noticably
- increase the speed of calculation, if the sample size is sufficiently high.
- If "auto" is selected, numba compatible fit functions get applied for data consisiting of > 200000 samples.
- eval_flags : bool, default True
- Wheather or not to assign new flags to the calculated residuals. If True, a residual gets assigned the worst
- flag present in the interval, the data for its calculation was obtained from.
- min_periods : {int, None}, default 0
- The minimum number of periods, that has to be available in every values fitting surrounding for the polynomial
- fit to be performed. If there are not enough values, np.nan gets assigned. Default (0) results in fitting
- regardless of the number of values present (results in overfitting for too sparse intervals). To automatically
- set the minimum number of periods to the number of values in an offset defined window size, pass np.nan.
- return_residues : bool, default False
- Internal parameter. Makes the method return the residues instead of the fit.
- """
- pass
-
diff --git a/docs/func_modules/drift.py b/docs/func_modules/drift.py
deleted file mode 100644
index 803bdec25..000000000
--- a/docs/func_modules/drift.py
+++ /dev/null
@@ -1,362 +0,0 @@
-"""
-
-"""
-def flagDriftFromNorm(field, fields, segment_freq, norm_spread, norm_frac, metric, linkage_method):
- """
- The function flags value courses that significantly deviate from a group of normal value courses.
-
- "Normality" is determined in terms of a maximum spreading distance, that members of a normal group must not exceed.
- In addition, only a group is considered "normal" if it contains more then `norm_frac` percent of the
- variables in "fields".
-
- See the Notes section for a more detailed presentation of the algorithm
-
- Parameters
- ----------
- field : str
- A dummy parameter.
- fields : str
- List of fieldnames in data, determining which variables are to be included into the flagging process.
- segment_freq : str
- An offset string, determining the size of the seperate datachunks that the algorihm is to be piecewise
- applied on.
- norm_spread : float
- A parameter limiting the maximum "spread" of the timeseries, allowed in the "normal" group. See Notes section
- for more details.
- norm_frac : float, default 0.5
- Has to be in [0,1]. Determines the minimum percentage of variables, the "normal" group has to comprise to be the
- normal group actually. The higher that value, the more stable the algorithm will be with respect to false
- positives. Also, nobody knows what happens, if this value is below 0.5.
- metric : Callable[[numpy.array, numpy.array], float]
- A distance function. It should be a function of 2 1-dimensional arrays and return a float scalar value.
- This value is interpreted as the distance of the two input arrays. The default is the averaged manhatten metric.
- See the Notes section to get an idea of why this could be a good choice.
- linkage_method : {"single", "complete", "average", "weighted", "centroid", "median", "ward"}, default "single"
- The linkage method used for hierarchical (agglomerative) clustering of the timeseries.
- See the Notes section for more details.
- The keyword gets passed on to scipy.hierarchy.linkage. See its documentation to learn more about the different
- keywords (References [1]).
- See wikipedia for an introduction to hierarchical clustering (References [2]).
- kwargs
-
- Notes
- -----
- following steps are performed for every data "segment" of length `segment_freq` in order to find the
- "abnormal" data:
-
- 1. Calculate the distances :math:`d(x_i,x_j)` for all :math:`x_i` in parameter `fields`. (with :math:`d`
- denoting the distance function
- passed to the parameter `metric`.
- 2. Calculate a dendogram with a hierarchical linkage algorithm, specified by the parameter `linkage_method`.
- 3. Flatten the dendogram at the level, the agglomeration costs exceed the value given by the parameter `norm_spread`
- 4. check if there is a cluster containing more than `norm_frac` percentage of the variables in fields.
-
- 1. if yes: flag all the variables that are not in that cluster (inside the segment)
- 2. if no: flag nothing
-
- The main parameter giving control over the algorithms behavior is the `norm_spread` parameter, that determines
- the maximum spread of a normal group by limiting the costs, a cluster agglomeration must not exceed in every
- linkage step.
- For singleton clusters, that costs just equal half the distance, the timeseries in the clusters, have to
- each other. So, no timeseries can be clustered together, that are more then
- 2*`norm_spread` distanted from each other.
- When timeseries get clustered together, this new clusters distance to all the other timeseries/clusters is
- calculated according to the linkage method specified by `linkage_method`. By default, it is the minimum distance,
- the members of the clusters have to each other.
- Having that in mind, it is advisable to choose a distance function, that can be well interpreted in the units
- dimension of the measurement and where the interpretation is invariant over the length of the timeseries.
- That is, why, the "averaged manhatten metric" is set as the metric default, since it corresponds to the
- averaged value distance, two timeseries have (as opposed by euclidean, for example).
-
- References
- ----------
- Documentation of the underlying hierarchical clustering algorithm:
- [1] https://docs.scipy.org/doc/scipy/reference/generated/scipy.cluster.hierarchy.linkage.html
- Introduction to Hierarchical clustering:
- [2] https://en.wikipedia.org/wiki/Hierarchical_clustering
- """
- pass
-
-
-def flagDriftFromReference(field, fields, segment_freq, thresh, metric):
- """
- The function flags value courses that deviate from a reference course by a margin exceeding a certain threshold.
-
- The deviation is measured by the distance function passed to parameter metric.
-
- Parameters
- ----------
- field : str
- The reference variable, the deviation from wich determines the flagging.
- fields : str
- List of fieldnames in data, determining wich variables are to be included into the flagging process.
- segment_freq : str
- An offset string, determining the size of the seperate datachunks that the algorihm is to be piecewise
- applied on.
- thresh : float
- The threshod by wich normal variables can deviate from the reference variable at max.
- metric : Callable[(numpyp.array, numpy-array), float]
- A distance function. It should be a function of 2 1-dimensional arrays and return a float scalar value.
- This value is interpreted as the distance of the two input arrays. The default is the averaged manhatten metric.
- See the Notes section to get an idea of why this could be a good choice.
- kwargs
-
- Notes
- -----
- it is advisable to choose a distance function, that can be well interpreted in the units
- dimension of the measurement and where the interpretation is invariant over the length of the timeseries.
- That is, why, the "averaged manhatten metric" is set as the metric default, since it corresponds to the
- averaged value distance, two timeseries have (as opposed by euclidean, for example).
- """
- pass
-
-
-def flagDriftFromScaledNorm(field, fields_scale1, fields_scale2, segment_freq, norm_spread, norm_frac, metric, linkage_method):
- """
- The function linearly rescales one set of variables to another set of variables with a different scale and then
- flags value courses that significantly deviate from a group of normal value courses.
-
- The two sets of variables can be linearly scaled one to another and hence the scaling transformation is performed
- via linear regression: A linear regression is performed on each pair of variables giving a slope and an intercept.
- The transformation is then calculated a the median of all the calculated slopes and intercepts.
-
- Once the transformation is performed, the function flags those values, that deviate from a group of normal values.
- "Normality" is determined in terms of a maximum spreading distance, that members of a normal group must not exceed.
- In addition, only a group is considered "normal" if it contains more then `norm_frac` percent of the
- variables in "fields".
-
- Parameters
- ----------
- field : str
- A dummy parameter.
- fields_scale1 : str
- List of fieldnames in data to be included into the flagging process which are scaled according to scaling
- scheme 1.
- fields_scale2 : str
- List of fieldnames in data to be included into the flagging process which are scaled according to scaling
- scheme 2.
- segment_freq : str
- An offset string, determining the size of the seperate datachunks that the algorihm is to be piecewise
- applied on.
- norm_spread : float
- A parameter limiting the maximum "spread" of the timeseries, allowed in the "normal" group. See Notes section
- for more details.
- norm_frac : float, default 0.5
- Has to be in [0,1]. Determines the minimum percentage of variables, the "normal" group has to comprise to be the
- normal group actually. The higher that value, the more stable the algorithm will be with respect to false
- positives. Also, nobody knows what happens, if this value is below 0.5.
- metric : Callable[(numpyp.array, numpy-array), float]
- A distance function. It should be a function of 2 1-dimensional arrays and return a float scalar value.
- This value is interpreted as the distance of the two input arrays. The default is the averaged manhatten metric.
- See the Notes section to get an idea of why this could be a good choice.
- linkage_method : {"single", "complete", "average", "weighted", "centroid", "median", "ward"}, default "single"
- The linkage method used for hierarchical (agglomerative) clustering of the timeseries.
- See the Notes section for more details.
- The keyword gets passed on to scipy.hierarchy.linkage. See its documentation to learn more about the different
- keywords (References [1]).
- See wikipedia for an introduction to hierarchical clustering (References [2]).
- kwargs
-
- References
- ----------
- Documentation of the underlying hierarchical clustering algorithm:
- [1] https://docs.scipy.org/doc/scipy/reference/generated/scipy.cluster.hierarchy.linkage.html
- Introduction to Hierarchical clustering:
- [2] https://en.wikipedia.org/wiki/Hierarchical_clustering
- """
- pass
-
-
-def correctExponentialDrift(field, maint_data_field, cal_mean, flag_maint_period):
- """
- The function fits an exponential model to chunks of data[field].
- It is assumed, that between maintenance events, there is a drift effect shifting the meassurements in a way, that
- can be described by the model M:
-
- M(t, a, b, c) = a + b(exp(c*t))
-
- Where as the values y_0 and y_1, describing the mean value directly after the last maintenance event (y_0) and
- directly before the next maintenance event (y_1), impose the following additional conditions on the drift model:.
-
- M(0, a, b, c) = y0
- M(1, a, b, c) = y1
-
- Solving the equation, one obtains the one-parameter Model:
-
- M_drift(t, c) = y0 + [(y1 - y0)/(exp(c) - )] * (exp(c*t) - 1)
-
- For every datachunk in between maintenance events.
-
- After having found the optimal parameter c*, the correction is performed by bending the fitted curve M_drift(t, c*),
- in a way that it matches y2 at t=1 (,with y2 being the mean value observed directly after the end of the next
- maintenance event).
- This bended curve is given by:
-
- M_shift(t, c*) = M(t, y0, [(y1 - y0)/(exp(c*) - )], c*)
-
- And the new values at t are computed via:
-
- new_vals(t) = old_vals(t) + M_shift(t) - M_drift(t)
-
- Parameters
- ----------
- field : str
- The fieldname of the data column, you want to correct.
- maint_data_field : str
- The fieldname of the datacolumn holding the maintenance information.
- The maint data is to expected to have following form:
- The series' timestamp itself represents the beginning of a
- maintenance event, wheras the values represent the endings of the maintenance intervals.
- cal_mean : int, default 5
- The number of values the mean is computed over, for obtaining the value level directly after and
- directly before maintenance event. This values are needed for shift calibration. (see above description)
- flag_maint_period : bool, default False
- Wheather or not to flag BAD the values directly obtained while maintenance.
- """
- pass
-
-
-def correctRegimeAnomaly(field, cluster_field, model, regime_transmission, x_date):
- """
- Function fits the passed model to the different regimes in data[field] and tries to correct
- those values, that have assigned a negative label by data[cluster_field].
-
- Currently, the only correction mode supported is the "parameter propagation."
-
- This means, any regime :math:`z`, labeled negatively and being modeled by the parameters p, gets corrected via:
-
- :math:`z_{correct} = z + (m(p^*) - m(p))`,
-
- where :math:`p^*` denotes the parameter set belonging to the fit of the nearest not-negatively labeled cluster.
-
- Parameters
- ----------
- field : str
- The fieldname of the data column, you want to correct.
- cluster_field : str
- A string denoting the field in data, holding the cluster label for the data you want to correct.
- model : Callable
- The model function to be fitted to the regimes.
- It must be a function of the form :math:`f(x, *p)`, where :math:`x` is the ``numpy.array`` holding the
- independent variables and :math:`p` are the model parameters that are to be obtained by fitting.
- Depending on the `x_date` parameter, independent variable x will either be the timestamps
- of every regime transformed to seconds from epoch, or it will be just seconds, counting the regimes length.
- regime_transmission : {None, str}, default None:
- If an offset string is passed, a data chunk of length `regime_transimission` right at the
- start and right at the end is ignored when fitting the model. This is to account for the
- unreliability of data near the changepoints of regimes.
- x_date : bool, default False
- If True, use "seconds from epoch" as x input to the model func, instead of "seconds from regime start".
-
- """
- pass
-
-
-def correctOffset():
- """
- Parameters
- ----------
- data : dios.DictOfSeries
- A dictionary of pandas.Series, holding all the data.
- field : str
- The fieldname of the data column, you want to correct.
- flagger : saqc.flagger
- A flagger object, holding flags and additional Informations related to `data`.
- max_mean_jump : float
- when searching for changepoints in mean - this is the threshold a mean difference in the
- sliding window search must exceed to trigger changepoint detection.
- normal_spread : float
- threshold denoting the maximum, regimes are allowed to abolutely differ in their means
- to form the "normal group" of values.
- search_winsz : str
- Size of the adjacent windows that are used to search for the mean changepoints.
- min_periods : int
- Minimum number of periods a search window has to contain, for the result of the changepoint
- detection to be considered valid.
- regime_transmission : {None, str}, default None:
- If an offset string is passed, a data chunk of length `regime_transimission` right from the
- start and right before the end of any regime is ignored when calculating a regimes mean for data correcture.
- This is to account for the unrelyability of data near the changepoints of regimes.
-
- """
- pass
-
-
-def flagRegimeAnomaly(field, cluster_field, norm_spread, linkage_method, metric, norm_frac):
- """
- A function to flag values belonging to an anomalous regime regarding modelling regimes of field.
-
- "Normality" is determined in terms of a maximum spreading distance, regimes must not exceed in respect
- to a certain metric and linkage method.
-
- In addition, only a range of regimes is considered "normal", if it models more then `norm_frac` percentage of
- the valid samples in "field".
-
- Note, that you must detect the regime changepoints prior to calling this function.
-
- Note, that it is possible to perform hypothesis tests for regime equality by passing the metric
- a function for p-value calculation and selecting linkage method "complete".
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged.
- cluster_field : str
- The name of the column in data, holding the cluster labels for the samples in field. (has to be indexed
- equal to field)
- norm_spread : float
- A threshold denoting the valuelevel, up to wich clusters a agglomerated.
- linkage_method : {"single", "complete", "average", "weighted", "centroid", "median", "ward"}, default "single"
- The linkage method used for hierarchical (agglomerative) clustering of the variables.
- metric : Callable[[numpy.array, numpy.array], float], default lambda x, y: np.abs(np.nanmean(x) - np.nanmean(y))
- A metric function for calculating the dissimilarity between 2 regimes. Defaults to just the difference in mean.
- norm_frac : float
- Has to be in [0,1]. Determines the minimum percentage of samples,
- the "normal" group has to comprise to be the normal group actually.
- """
- pass
-
-
-def assignRegimeAnomaly(field, cluster_field, norm_spread, linkage_method, metric, norm_frac, set_cluster, set_flags):
- """
- A function to detect values belonging to an anomalous regime regarding modelling regimes of field.
-
- The function changes the value of the regime cluster labels to be negative.
-
- "Normality" is determined in terms of a maximum spreading distance, regimes must not exceed in respect
- to a certain metric and linkage method.
-
- In addition, only a range of regimes is considered "normal", if it models more then `norm_frac` percentage of
- the valid samples in "field".
-
- Note, that you must detect the regime changepoints prior to calling this function. (They are expected to be stored
- parameter `cluster_field`.)
-
- Note, that it is possible to perform hypothesis tests for regime equality by passing the metric
- a function for p-value calculation and selecting linkage method "complete".
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged.
- cluster_field : str
- The name of the column in data, holding the cluster labels for the samples in field. (has to be indexed
- equal to field)
- norm_spread : float
- A threshold denoting the valuelevel, up to wich clusters a agglomerated.
- linkage_method : {"single", "complete", "average", "weighted", "centroid", "median", "ward"}, default "single"
- The linkage method used for hierarchical (agglomerative) clustering of the variables.
- metric : Callable[[numpy.array, numpy.array], float], default lambda x, y: np.abs(np.nanmean(x) - np.nanmean(y))
- A metric function for calculating the dissimilarity between 2 regimes. Defaults to just the difference in mean.
- norm_frac : float
- Has to be in [0,1]. Determines the minimum percentage of samples,
- the "normal" group has to comprise to be the normal group actually.
- set_cluster : bool, default False
- If True, all data, considered "anormal", gets assigned a negative clusterlabel. This option
- is present for further use (correction) of the anomaly information.
- set_flags : bool, default True
- Wheather or not to flag abnormal values (do not flag them, if you want to correct them
- afterwards, becasue flagged values usually are not visible in further tests.).
- """
- pass
-
diff --git a/docs/func_modules/flagtools.py b/docs/func_modules/flagtools.py
deleted file mode 100644
index 9809d7a1c..000000000
--- a/docs/func_modules/flagtools.py
+++ /dev/null
@@ -1,135 +0,0 @@
-"""
-
-"""
-def flagDummy(field):
- """
- Function does nothing but returning data and flagger.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged.
- """
- pass
-
-
-def flagForceFail(field):
- """
- Function raises a runtime error.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged.
- """
- pass
-
-
-def flagUnflagged(field, kwargs):
- """
- Function sets the flagger.GOOD flag to all values flagged better then flagger.GOOD.
- If there is an entry 'flag' in the kwargs dictionary passed, the
- function sets the kwargs['flag'] flag to all values flagged better kwargs['flag']
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged.
- kwargs : Dict
- If kwargs contains 'flag' entry, kwargs['flag] is set, if no entry 'flag' is present,
- 'flagger.UNFLAGGED' is set.
- """
- pass
-
-
-def flagGood(field):
- """
- Function sets the flagger.GOOD flag to all values flagged better then flagger.GOOD.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged.
- """
- pass
-
-
-def flagManual(field, mdata, mflag, method):
- """
- Flag data by given, "manually generated" data.
-
- The data is flagged at locations where `mdata` is equal to a provided flag (`mflag`).
- The format of mdata can be an indexed object, like pd.Series, pd.Dataframe or dios.DictOfSeries,
- but also can be a plain list- or array-like.
- How indexed mdata is aligned to data is specified via the `method` parameter.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged.
- mdata : {pd.Series, pd.Dataframe, DictOfSeries}
- The "manually generated" data
- mflag : scalar
- The flag that indicates data points in `mdata`, of wich the projection in data should be flagged.
- method : {'plain', 'ontime', 'left-open', 'right-open'}, default plain
- Defines how mdata is projected on data. Except for the 'plain' method, the methods assume mdata to have an
- index.
-
- * 'plain': mdata must have the same length as data and is projected one-to-one on data.
- * 'ontime': works only with indexed mdata. mdata entries are matched with data entries that have the same index.
- * 'right-open': mdata defines intervals, values are to be projected on.
- The intervals are defined by any two consecutive timestamps t_1 and 1_2 in mdata.
- the value at t_1 gets projected onto all data timestamps t with t_1 <= t < t_2.
- * 'left-open': like 'right-open', but the projected interval now covers all t with t_1 < t <= t_2.
-
- Examples
- --------
- An example for mdata
- >>> mdata = pd.Series([1,0,1], index=pd.to_datetime(['2000-02', '2000-03', '2001-05']))
- >>> mdata
- 2000-02-01 1
- 2000-03-01 0
- 2001-05-01 1
- dtype: int64
-
- On *dayly* data, with the 'ontime' method, only the provided timestamnps are used.
- Bear in mind that only exact timestamps apply, any offset will result in ignoring
- the timestamp.
- >>> _, fl = flagManual(data, field, flagger, mdata, mflag=1, method='ontime')
- >>> fl.isFlagged(field)
- 2000-01-31 False
- 2000-02-01 True
- 2000-02-02 False
- 2000-02-03 False
- .. ..
- 2000-02-29 False
- 2000-03-01 True
- 2000-03-02 False
- Freq: D, dtype: bool
-
- With the 'right-open' method, the mdata is forward fill:
- >>> _, fl = flagManual(data, field, flagger, mdata, mflag=1, method='right-open')
- >>> fl.isFlagged(field)
- 2000-01-31 False
- 2000-02-01 True
- 2000-02-02 True
- .. ..
- 2000-02-29 True
- 2000-03-01 False
- 2000-03-02 False
- Freq: D, dtype: bool
-
- With the 'left-open' method, backward filling is used:
- >>> _, fl = flagManual(data, field, flagger, mdata, mflag=1, method='left-open')
- >>> fl.isFlagged(field)
- 2000-01-31 False
- 2000-02-01 False
- 2000-02-02 True
- .. ..
- 2000-02-29 True
- 2000-03-01 True
- 2000-03-02 False
- Freq: D, dtype: bool
- """
- pass
-
diff --git a/docs/func_modules/generic.py b/docs/func_modules/generic.py
deleted file mode 100644
index 693fde4fa..000000000
--- a/docs/func_modules/generic.py
+++ /dev/null
@@ -1,108 +0,0 @@
-"""
-
-"""
-def process(field, func, nodata):
- """
- generate/process data with generically defined functions.
-
- The functions can depend on on any of the fields present in data.
-
- Formally, what the function does, is the following:
-
- 1. Let F be a Callable, depending on fields f_1, f_2,...f_K, (F = F(f_1, f_2,...f_K))
- Than, for every timestamp t_i that occurs in at least one of the timeseries data[f_j] (outer join),
- The value v_i is computed via:
- v_i = data([f_1][t_i], data[f_2][t_i], ..., data[f_K][t_i]), if all data[f_j][t_i] do exist
- v_i = `nodata`, if at least one of the data[f_j][t_i] is missing.
- 2. The result is stored to data[field] (gets generated if not present)
-
- Parameters
- ----------
- field : str
- The fieldname of the column, where you want the result from the generic expressions processing to be written to.
- func : Callable
- The data processing function with parameter names that will be
- interpreted as data column entries.
- See the examples section to learn more.
- nodata : any, default np.nan
- The value that indicates missing/invalid data
-
- Examples
- --------
- Some examples on what to pass to the func parameter:
- To compute the sum of the variables "temperature" and "uncertainty", you would pass the function:
-
- >>> lambda temperature, uncertainty: temperature + uncertainty
-
- You also can pass numpy and pandas functions:
-
- >>> lambda temperature, uncertainty: np.round(temperature) * np.sqrt(uncertainty)
- """
- pass
-
-
-def flag(field, func, nodata):
- """
- a function to flag a data column by evaluation of a generic expression.
-
- The expression can depend on any of the fields present in data.
-
- Formally, what the function does, is the following:
-
- Let X be an expression, depending on fields f_1, f_2,...f_K, (X = X(f_1, f_2,...f_K))
- Than for every timestamp t_i in data[field]:
- data[field][t_i] is flagged if X(data[f_1][t_i], data[f_2][t_i], ..., data[f_K][t_i]) is True.
-
- Note, that all value series included in the expression to evaluate must be labeled identically to field.
-
- Note, that the expression is passed in the form of a Callable and that this callables variable names are
- interpreted as actual names in the data header. See the examples section to get an idea.
-
- Note, that all the numpy functions are available within the generic expressions.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, where you want the result from the generic expressions evaluation to be projected
- to.
- func : Callable
- The expression that is to be evaluated is passed in form of a callable, with parameter names that will be
- interpreted as data column entries. The Callable must return an boolen array like.
- See the examples section to learn more.
- nodata : any, default np.nan
- The value that indicates missing/invalid data
-
- Examples
- --------
- Some examples on what to pass to the func parameter:
- To flag the variable `field`, if the sum of the variables
- "temperature" and "uncertainty" is below zero, you would pass the function:
-
- >>> lambda temperature, uncertainty: temperature + uncertainty < 0
-
- There is the reserved name 'This', that always refers to `field`. So, to flag field if field is negative, you can
- also pass:
-
- >>> lambda this: this < 0
-
- If you want to make dependent the flagging from flags already present in the data, you can use the built-in
- ``isflagged`` method. For example, to flag the 'temperature', if 'level' is flagged, you would use:
-
- >>> lambda level: isflagged(level)
-
- You can furthermore specify a flagging level, you want to compare the flags to. For example, for flagging
- 'temperature', if 'level' is flagged at a level named 'doubtfull' or worse, use:
-
- >>> lambda level: isflagged(level, flag='doubtfull', comparator='<=')
-
- If you are unsure about the used flaggers flagging level names, you can use the reserved key words BAD, UNFLAGGED
- and GOOD, to refer to the worst (BAD), best(GOOD) or unflagged (UNFLAGGED) flagging levels. For example.
-
- >>> lambda level: isflagged(level, flag=UNFLAGGED, comparator='==')
-
- Your expression also is allowed to include pandas and numpy functions
-
- >>> lambda level: np.sqrt(level) > 7
- """
- pass
-
diff --git a/docs/func_modules/interpolation.py b/docs/func_modules/interpolation.py
deleted file mode 100644
index 9009d87c8..000000000
--- a/docs/func_modules/interpolation.py
+++ /dev/null
@@ -1,123 +0,0 @@
-"""
-
-"""
-def interpolateByRolling(field, winsz, func, center, min_periods, interpol_flag):
- """
- Interpolates missing values (nan values present in the data) by assigning them the aggregation result of
- a window surrounding them.
-
- Note, that in the current implementation, center=True can only be used with integer window sizes - furthermore
- note, that integer window sizes can yield screwed aggregation results for not-harmonized or irregular data.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-interpolated.
- winsz : int, str
- The size of the window, the aggregation is computed from. Either counted in periods number (Integer passed),
- or defined by a total temporal extension (offset String passed).
- func : Callable
- The function used for aggregation.
- center : bool, default True
- Wheather or not the window, the aggregation is computed of, is centered around the value to be interpolated.
- min_periods : int
- Minimum number of valid (not np.nan) values that have to be available in a window for its aggregation to be
- computed.
- interpol_flag : {'GOOD', 'BAD', 'UNFLAGGED', str}, default 'UNFLAGGED'
- Flag that is to be inserted for the interpolated values. You can either pass one of the three major flag-classes
- or specify directly a certain flag from the passed flagger.
- """
- pass
-
-
-def interpolateInvalid(field, method, inter_order, inter_limit, interpol_flag, downgrade_interpolation, not_interpol_flags):
- """
- Function to interpolate nan values in the data.
-
- There are available all the interpolation methods from the pandas.interpolate method and they are applicable by
- the very same key words, that you would pass to the ``pd.Series.interpolate``'s method parameter.
-
- Note, that the `inter_limit` keyword really restricts the interpolation to chunks, not containing more than
- `inter_limit` successive nan entries.
-
- Note, that the function differs from ``proc_interpolateGrid``, in its behaviour to ONLY interpolate nan values that
- were already present in the data passed.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-interpolated.
- method : {"linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric",
- "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"}: string
- The interpolation method you want to apply.
- inter_order : int, default 2
- If there your selected interpolation method can be performed at different 'orders' - here you pass the desired
- order.
- inter_limit : int, default 2
- Maximum number of consecutive 'nan' values allowed for a gap to be interpolated.
- interpol_flag : {'GOOD', 'BAD', 'UNFLAGGED', str}, default 'UNFLAGGED'
- Flag that is to be inserted for the interpolated values. You can either pass one of the three major flag-classes
- or specify directly a certain flag from the passed flagger.
- downgrade_interpolation : bool, default False
- If interpolation can not be performed at `inter_order` - (not enough values or not implemented at this order) -
- automaticalyy try to interpolate at order `inter_order` :math:`- 1`.
- not_interpol_flags : {None, str, List[str]}, default None
- A list of flags or a single Flag, marking values, you want NOT to be interpolated.
- """
- pass
-
-
-def interpolateIndex(field, freq, method, inter_order, to_drop, downgrade_interpolation, empty_intervals_flag, grid_field, inter_limit, freq_check):
- """
- Function to interpolate the data at regular (equidistant) timestamps (or Grid points).
-
- Note, that the interpolation will only be calculated, for grid timestamps that have a preceding AND a succeeding
- valid data value within "freq" range.
-
- Note, that the function differs from proc_interpolateMissing, by returning a whole new data set, only containing
- samples at the interpolated, equidistant timestamps (of frequency "freq").
-
- Note, it is possible to interpolate unregular "grids" (with no frequencies). In fact, any date index
- can be target of the interpolation. Just pass the field name of the variable, holding the index
- you want to interpolate, to "grid_field". 'freq' is then use to determine the maximum gap size for
- a grid point to be interpolated.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-interpolated.
- freq : str
- An Offset String, interpreted as the frequency of
- the grid you want to interpolate your data at.
- method : {"linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric",
- "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"}: string
- The interpolation method you want to apply.
- inter_order : integer, default 2
- If there your selected interpolation method can be performed at different 'orders' - here you pass the desired
- order.
- to_drop : {None, str, List[str]}, default None
- Flags that refer to values you want to drop before interpolation - effectively excluding grid points from
- interpolation, that are only surrounded by values having a flag in them, that is listed in drop flags. Default
- results in the flaggers *BAD* flag to be the drop_flag.
- downgrade_interpolation : bool, default False
- If interpolation can not be performed at `inter_order` - (not enough values or not implemented at this order) -
- automatically try to interpolate at order `inter_order` :math:`- 1`.
- empty_intervals_flag : str, default None
- A Flag, that you want to assign to those values in the resulting equidistant sample grid, that were not
- surrounded by valid data in the original dataset, and thus were not interpolated. Default automatically assigns
- ``flagger.BAD`` flag to those values.
- grid_field : String, default None
- Use the timestamp of another variable as (not necessarily regular) "grid" to be interpolated.
- inter_limit : Integer, default 2
- Maximum number of consecutive Grid values allowed for interpolation. If set
- to *n*, chunks of *n* and more consecutive grid values, where there is no value in between, wont be
- interpolated.
- freq_check : {None, 'check', 'auto'}, default None
-
- * ``None``: do not validate frequency-string passed to `freq`
- * ``'check'``: estimate frequency and log a warning if estimate miss matchs frequency string passed to 'freq', or
- if no uniform sampling rate could be estimated
- * ``'auto'``: estimate frequency and use estimate. (Ignores `freq` parameter.)
- """
- pass
-
diff --git a/docs/func_modules/module_dict.pkl b/docs/func_modules/module_dict.pkl
deleted file mode 100644
index 7fafca46813d0491babe42170df526c3842f39ab..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001
literal 2466
zcmaJ@=~vrE5O<p9A|dyERZWvtv{dPR&=W(TB;}|fuszhUNMn18q}A+7y!b=k>!04y
zNVX*4<&(X?`OUF&t^PatFLqpUy3^_G8Ib!HPx$zjE3QreuOG?5s<Kuk13W3NwFN6y
zM>2yD-xinKlIMAxT08~f(gc4M1~xN=ozvp{iB4w`G6*s{JM3H8p%V;cG63tS#K}|)
z7L20PfsAr+0-YNR)|;Bz8Y=6=I?N+*Qlk%4NThzAkTIQDC(};IrX87tP7%LzyoVt;
zq0D5<{;U(JeQ*XcwNvg~I+`o&&h0j`usbvB{8QZSX=rq6okJ|X`0k9OjiajUln#zC
z$I08B@$iheKzS;!pK6t47SH;4&J_<1!Ca38PSZxon-WhBb0n2(kTh5eQEoFZcpk)!
zL*%+D7X*nqI4#Z<6v|{Ki#RU&c*zyFnphdcgmGEiJ|gXowxv-r2>`DE_tV5;?eXex
zTAVFZb8`TFmEkoXuZx=v#3OV=N25f?N=A4?iYpCiZRsXiIz^JusD6gGe7s$T%x4aF
zv5+ho9j|E<;~jB-%x+72&<6uW*mpVi#~9cQWCS6bx!2$X+8FBRXRlM8K$20o>=N&b
zyAxLxrw0S-?s>YDj7Io?r>b;R5!U;C3mHD-wAUfD_1+P8CWgz3x8)9qi(^8asdp(;
zMS+Ws&a{p!KJxKBmt&Y<Wlr)q9^qq-p)Fa`CV(ZWB7Aa;^kq`82ecL5=fbyjm$eS@
z1980>0=rw4Nlqnp%Wjz{)sfFo<Y!220154d06+BcBbWC}g?O|K0<acmd78#(VG2J6
z-cS`*8}-QOG#RCl1CtipjCKe<_3;x|JUWbfH4D0<o{pkon|&&rHxXA7+FvTf&&0zw
z(?WQSp98ml?T4bNgZ+YnE#^h%S){;N{L;s-T+Zbf;#77=o0*X#{F<k|C0gCyq#gr{
z-*9bOvYu2H@LNt)ORze>h;!T_Ex78^?~Z!uj+RZnw`cMD<Gj#{#~;MK#zi>l1*5Iq
z%%}o3qw@X;JnLhRbrycA`XH4V)hxlEeVliB1yu-J+bE^o(c;z)c6r>774OIch-lUp
z$_A-8wYNp`HbtRQ&Sac&!JqlKRA#*fJ?eM^19}W`nd5GVNC~9DU%0dlysk3f3SXch
zjAgo|*Nqyegr?#-XTKq?{sO+>32aGJLanq!U2pMMIHXu#Y_)18QdMD29kJ%)x_De(
zl@X5?U62<Ke7%BuWA;Y5ipScu-QTM6m})$FK5p_nR)AOh;+Et#tFT)8C21F*Bev|)
zZ69~Iy~?v!?ztChYuv>sB+q{-td0CmMvB*<+;4yQ_@~P?<g*v(bcBC3fZ`oTpJOZ;
z^Z>{h@b3mEmUbUsl^c~wwYOdJRxDZ?xy1JAbt!j<^l^`i+Qz-f5d+D!n*gj=+YN`)
Gx%WRP@H;&K
diff --git a/docs/func_modules/outliers.py b/docs/func_modules/outliers.py
deleted file mode 100644
index 183cfc24a..000000000
--- a/docs/func_modules/outliers.py
+++ /dev/null
@@ -1,359 +0,0 @@
-"""
-
-"""
-def flagByStray(field, partition_freq, partition_min, iter_start, alpha):
- """
- Flag outliers in 1-dimensional (score) data with the STRAY Algorithm.
-
- Find more information on the algorithm in References [1].
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged.
- partition_freq : {None, str, int}, default None
- partition_freq : {np.inf, float, str}, default np.inf
- Determines the segmentation of the data into partitions, the kNN algorithm is
- applied onto individually.
-
- * ``np.inf``: Apply Scoring on whole data set at once
- * ``x`` > 0 : Apply scoring on successive data chunks of periods length ``x``
- * Offset String : Apply scoring on successive partitions of temporal extension matching the passed offset
- string
-
- partition_min : int, default 11
- Minimum number of periods per partition that have to be present for a valid outlier dettection to be made in
- this partition. (Only of effect, if `partition_freq` is an integer.) Partition min value must always be
- greater then the nn_neighbors value.
- iter_start : float, default 0.5
- Float in [0,1] that determines which percentage of data is considered "normal". 0.5 results in the stray
- algorithm to search only the upper 50 % of the scores for the cut off point. (See reference section for more
- information)
- alpha : float, default 0.05
- Level of significance by which it is tested, if a score might be drawn from another distribution, than the
- majority of the data.
-
- References
- ----------
- [1] Talagala, P. D., Hyndman, R. J., & Smith-Miles, K. (2019). Anomaly detection in high dimensional data.
- arXiv preprint arXiv:1908.04000.
- """
- pass
-
-
-def flagMVScores(field, fields, trafo, alpha, n_neighbors, scoring_func, iter_start, stray_partition, stray_partition_min, trafo_on_partition, reduction_range, reduction_drop_flagged, reduction_thresh, reduction_min_periods):
- """
- The algorithm implements a 3-step outlier detection procedure for simultaneously flagging of higher dimensional
- data (dimensions > 3).
-
- In references [1], the procedure is introduced and exemplified with an application on hydrological data.
-
- See the notes section for an overview over the algorithms basic steps.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged. (Here a dummy, for structural reasons)
- fields : List[str]
- List of fieldnames, corresponding to the variables that are to be included into the flagging process.
- trafo : callable, default lambda x:x
- Transformation to be applied onto every column before scoring. Will likely get deprecated soon. Its better
- to transform the data in a processing step, preceeeding the call to ``flagMVScores``.
- alpha : float, default 0.05
- Level of significance by which it is tested, if an observations score might be drawn from another distribution
- than the majority of the observation.
- n_neighbors : int, default 10
- Number of neighbors included in the scoring process for every datapoint.
- scoring_func : Callable[numpy.array, float], default np.sum
- The function that maps the set of every points k-nearest neighbor distances onto a certain scoring.
- iter_start : float, default 0.5
- Float in [0,1] that determines which percentage of data is considered "normal". 0.5 results in the threshing
- algorithm to search only the upper 50 % of the scores for the cut off point. (See reference section for more
- information)
- stray_partition : {None, str, int}, default None
- Only effective when `threshing` = 'stray'.
- Determines the size of the data partitions, the data is decomposed into. Each partition is checked seperately
- for outliers. If a String is passed, it has to be an offset string and it results in partitioning the data into
- parts of according temporal length. If an integer is passed, the data is simply split up into continous chunks
- of `partition_freq` periods. if ``None`` is passed (default), all the data will be tested in one run.
- stray_partition_min : int, default 11
- Only effective when `threshing` = 'stray'.
- Minimum number of periods per partition that have to be present for a valid outlier detection to be made in
- this partition. (Only of effect, if `stray_partition` is an integer.)
- trafo_on_partition : bool, default True
- Whether or not to apply the passed transformation on every partition the algorithm is applied on, separately.
- reduction_range : {None, str}, default None
- If not None, it is tried to reduce the stray result onto single outlier components of the input fields.
- An offset string, denoting the range of the temporal surrounding to include into the MAD testing while trying
- to reduce flags.
- reduction_drop_flagged : bool, default False
- Only effective when `reduction_range` is not ``None``.
- Whether or not to drop flagged values other than the value under test from the temporal surrounding
- before checking the value with MAD.
- reduction_thresh : float, default 3.5
- Only effective when `reduction_range` is not ``None``.
- The `critical` value, controlling wheather the MAD score is considered referring to an outlier or not.
- Higher values result in less rigid flagging. The default value is widely considered apropriate in the
- literature.
- reduction_min_periods : int, 1
- Only effective when `reduction_range` is not ``None``.
- Minimum number of meassurements necessarily present in a reduction interval for reduction actually to be
- performed.
-
- Notes
- -----
- The basic steps are:
-
- 1. transforming
-
- The different data columns are transformed via timeseries transformations to
- (a) make them comparable and
- (b) make outliers more stand out.
-
- This step is usually subject to a phase of research/try and error. See [1] for more details.
-
- Note, that the data transformation as an built-in step of the algorithm, will likely get deprecated soon. Its better
- to transform the data in a processing step, preceeding the multivariate flagging process. Also, by doing so, one
- gets mutch more control and variety in the transformation applied, since the `trafo` parameter only allows for
- application of the same transformation to all of the variables involved.
-
- 2. scoring
-
- Every observation gets assigned a score depending on its k nearest neighbors. See the `scoring_method` parameter
- description for details on the different scoring methods. Furthermore [1], [2] may give some insight in the
- pro and cons of the different methods.
-
- 3. threshing
-
- The gaps between the (greatest) scores are tested for beeing drawn from the same
- distribution as the majority of the scores. If a gap is encountered, that, with sufficient significance, can be
- said to not be drawn from the same distribution as the one all the smaller gaps are drawn from, than
- the observation belonging to this gap, and all the observations belonging to gaps larger then this gap, get flagged
- outliers. See description of the `threshing` parameter for more details. Although [2] gives a fully detailed
- overview over the `stray` algorithm.
- """
- pass
-
-
-def flagRaise(field, thresh, raise_window, intended_freq, average_window, mean_raise_factor, min_slope, min_slope_weight, numba_boost):
- """
- The function flags raises and drops in value courses, that exceed a certain threshold
- within a certain timespan.
-
- The parameter variety of the function is owned to the intriguing
- case of values, that "return" from outlierish or anomalious value levels and
- thus exceed the threshold, while actually being usual values.
-
- NOTE, the dataset is NOT supposed to be harmonized to a time series with an
- equidistant frequency grid.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged.
- thresh : float
- The threshold, for the total rise (thresh > 0), or total drop (thresh < 0), value courses must
- not exceed within a timespan of length `raise_window`.
- raise_window : str
- An offset string, determining the timespan, the rise/drop thresholding refers to. Window is inclusively defined.
- intended_freq : str
- An offset string, determining The frequency, the timeseries to-be-flagged is supposed to be sampled at.
- The window is inclusively defined.
- average_window : {None, str}, default None
- See condition (2) of the description linked in the references. Window is inclusively defined.
- The window defaults to 1.5 times the size of `raise_window`
- mean_raise_factor : float, default 2
- See second condition listed in the notes below.
- min_slope : {None, float}, default None
- See third condition listed in the notes below.
- min_slope_weight : float, default 0.8
- See third condition listed in the notes below.
- numba_boost : bool, default True
-
- Notes
- -----
- The value :math:`x_{k}` of a time series :math:`x` with associated
- timestamps :math:`t_i`, is flagged a raise, if:
-
- * There is any value :math:`x_{s}`, preceeding :math:`x_{k}` within `raise_window` range, so that:
-
- * :math:`M = |x_k - x_s | >` `thresh` :math:`> 0`
-
- * The weighted average :math:`\mu^{*}` of the values, preceding :math:`x_{k}` within `average_window`
- range indicates, that :math:`x_{k}` does not return from an "outlierish" value course, meaning that:
-
- * :math:`x_k > \mu^* + ( M` / `mean_raise_factor` :math:`)`
-
- * Additionally, if `min_slope` is not `None`, :math:`x_{k}` is checked for being sufficiently divergent from its
- very predecessor :max:`x_{k-1}`$, meaning that, it is additionally checked if:
-
- * :math:`x_k - x_{k-1} >` `min_slope`
- * :math:`t_k - t_{k-1} >` `min_slope_weight` :math:`\times` `intended_freq`
- """
- pass
-
-
-def flagMAD(field, window):
- """
- The function represents an implementation of the modyfied Z-score outlier detection method.
-
- See references [1] for more details on the algorithm.
-
- Note, that the test needs the input data to be sampled regularly (fixed sampling rate).
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged. (Here a dummy, for structural reasons)
- window : str
- Offset string. Denoting the windows size that the "Z-scored" values have to lie in.
- z: float, default 3.5
- The value the Z-score is tested against. Defaulting to 3.5 (Recommendation of [1])
-
- References
- ----------
- [1] https://www.itl.nist.gov/div898/handbook/eda/section3/eda35h.htm
- """
- pass
-
-
-def flagOffset(field, thresh, tolerance, window, numba_kickin):
- """
- A basic outlier test that is designed to work for harmonized and not harmonized data.
-
- The test classifies values/value courses as outliers by detecting not only a rise in value, but also,
- checking for a return to the initial value level.
-
- Values :math:`x_n, x_{n+1}, .... , x_{n+k}` of a timeseries :math:`x` with associated timestamps
- :math:`t_n, t_{n+1}, .... , t_{n+k}` are considered spikes, if
-
- 1. :math:`|x_{n-1} - x_{n + s}| >` `thresh`, for all :math:`s \in [0,1,2,...,k]`
-
- 2. :math:`|x_{n-1} - x_{n+k+1}| <` `tolerance`
-
- 3. :math:`|t_{n-1} - t_{n+k+1}| <` `window`
-
- Note, that this definition of a "spike" not only includes one-value outliers, but also plateau-ish value courses.
-
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged. (Here a dummy, for structural reasons)
- thresh : float
- Minimum difference between to values, to consider the latter one as a spike. See condition (1)
- tolerance : float
- Maximum difference between pre-spike and post-spike values. See condition (2)
- window : {str, int}, default '15min'
- Maximum length of "spiky" value courses. See condition (3). Integer defined window length are only allowed for
- regularly sampled timeseries.
- numba_kickin : int, default 200000
- When there are detected more than `numba_kickin` incidents of potential spikes,
- the pandas.rolling - part of computation gets "jitted" with numba.
- Default value hast proven to be around the break even point between "jit-boost" and "jit-costs".
-
-
- References
- ----------
- The implementation is a time-window based version of an outlier test from the UFZ Python library,
- that can be found here:
-
- https://git.ufz.de/chs/python/blob/master/ufz/level1/spike.py
- """
- pass
-
-
-def flagByGrubbs(field, winsz, alpha, min_periods):
- """
- The function flags values that are regarded outliers due to the grubbs test.
-
- See reference [1] for more information on the grubbs tests definition.
-
- The (two-sided) test gets applied onto data chunks of size "winsz". The tests application will
- be iterated on each data-chunk under test, till no more outliers are detected in that chunk.
-
- Note, that the test performs poorely for small data chunks (resulting in heavy overflagging).
- Therefor you should select "winsz" so that every window contains at least > 8 values and also
- adjust the min_periods values accordingly.
-
- Note, that the data to be tested by the grubbs test are expected to be distributed "normalish".
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged.
- winsz : {int, str}
- The size of the window you want to use for outlier testing. If an integer is passed, the size
- refers to the number of periods of every testing window. If a string is passed, it has to be an offset string,
- and will denote the total temporal extension of every window.
- alpha : float, default 0.05
- The level of significance, the grubbs test is to be performed at. (between 0 and 1)
- min_periods : int, default 8
- The minimum number of values that have to be present in an interval under test, for a grubbs test result to be
- accepted. Only makes sence in case `winsz` is an offset string.
- check_lagged: boolean, default False
- If True, every value gets checked twice for being an outlier. Ones in the initial rolling window and one more
- time in a rolling window that is lagged by half the windows delimeter (winsz/2). Recommended for avoiding false
- positives at the window edges. Only available when rolling with integer defined window size.
-
- References
- ----------
- introduction to the grubbs test:
-
- [1] https://en.wikipedia.org/wiki/Grubbs%27s_test_for_outliers
- """
- pass
-
-
-def flagRange(field, min, max):
- """
- Function flags values not covered by the closed interval [`min`, `max`].
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged.
- min : float
- Lower bound for valid data.
- max : float
- Upper bound for valid data.
- """
- pass
-
-
-def flagCrossStatistic(field, fields, thresh, cross_stat):
- """
- Function checks for outliers relatively to the "horizontal" input data axis.
-
- For `fields` :math:`=[f_1,f_2,...,f_N]` and timestamps :math:`[t_1,t_2,...,t_K]`, the following steps are taken
- for outlier detection:
-
- 1. All timestamps :math:`t_i`, where there is one :math:`f_k`, with :math:`data[f_K]` having no entry at
- :math:`t_i`, are excluded from the following process (inner join of the :math:`f_i` fields.)
- 2. for every :math:`0 <= i <= K`, the value
- :math:`m_j = median(\{data[f_1][t_i], data[f_2][t_i], ..., data[f_N][t_i]\})` is calculated
- 2. for every :math:`0 <= i <= K`, the set
- :math:`\{data[f_1][t_i] - m_j, data[f_2][t_i] - m_j, ..., data[f_N][t_i] - m_j\}` is tested for outliers with the
- specified method (`cross_stat` parameter).
-
- Parameters
- ----------
- field : str
- A dummy parameter.
- fields : str
- List of fieldnames in data, determining wich variables are to be included into the flagging process.
- thresh : float
- Threshold which the outlier score of an value must exceed, for being flagged an outlier.
- cross_stat : {'modZscore', 'Zscore'}, default 'modZscore'
- Method used for calculating the outlier scores.
-
- * ``'modZscore'``: Median based "sigma"-ish approach. See Referenecs [1].
- * ``'Zscore'``: Score values by how many times the standard deviation they differ from the median.
- See References [1]
-
- References
- ----------
- [1] https://www.itl.nist.gov/div898/handbook/eda/section3/eda35h.htm
- """
- pass
-
diff --git a/docs/func_modules/pattern.py b/docs/func_modules/pattern.py
deleted file mode 100644
index 13e769ac6..000000000
--- a/docs/func_modules/pattern.py
+++ /dev/null
@@ -1,38 +0,0 @@
-"""
-
-"""
-def flagPatternByDTW(field):
- """
- Pattern recognition via wavelets.
-
- The steps are:
- 1. work on chunks returned by a moving window
- 2. each chunk is compared to the given pattern, using the wavelet algorithm as presented in [1]
- 3. if the compared chunk is equal to the given pattern it gets flagged
-
- Parameters
- ----------
-
- field : str
- The fieldname of the data column, you want to correct.
- """
- pass
-
-
-def flagPatternByWavelet(field):
- """
- Pattern Recognition via Dynamic Time Warping.
-
- The steps are:
- 1. work on chunks returned by a moving window
- 2. each chunk is compared to the given pattern, using the dynamic time warping algorithm as presented in [1]
- 3. if the compared chunk is equal to the given pattern it gets flagged
-
- Parameters
- ----------
-
- field : str
- The fieldname of the data column, you want to correct.
- """
- pass
-
diff --git a/docs/func_modules/resampling.py b/docs/func_modules/resampling.py
deleted file mode 100644
index 3338e6534..000000000
--- a/docs/func_modules/resampling.py
+++ /dev/null
@@ -1,304 +0,0 @@
-"""
-
-"""
-def aggregate(field, freq, value_func, flag_func, method, to_drop):
- """
- A method to "regularize" data by aggregating (resampling) data at a regular timestamp.
-
- A series of data is considered "regular", if it is sampled regularly (= having uniform sampling rate).
-
- The data will therefor get aggregated with a function, specified by the `value_func` parameter and
- the result gets projected onto the new timestamps with a method, specified by "method".
-
- The following method (keywords) are available:
-
- * ``'nagg'``: (aggreagtion to nearest) - all values in the range (+/- freq/2) of a grid point get aggregated with
- `agg_func`. and assigned to it. Flags get aggregated by `flag_func` and assigned the same way.
- * ``'bagg'``: (backwards aggregation) - all values in a sampling interval get aggregated with agg_func and the
- result gets assigned to the last regular timestamp. Flags get aggregated by `flag_func` and assigned the same way.
- * ``'fagg'``: (forward aggregation) - all values in a sampling interval get aggregated with agg_func and the result
- gets assigned to the next regular timestamp. Flags get aggregated by `flag_func` and assigned the same way.
-
- Note, that, if there is no valid data (exisitng and not-na) available in a sampling interval assigned to a regular
- timestamp by the selected method, nan gets assigned to this timestamp. The associated flag will be of value
- ``flagger.UNFLAGGED``.
-
- Note: the method will likely and significantly alter values and shape of ``data[field]``. The original data is kept
- in the data dios and assigned to the fieldname ``field + '_original'``.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-regularized.
- freq : str
- The sampling frequency the data is to be aggregated (resampled) at.
- value_func : Callable
- The function you want to use for aggregation.
- flag_func : Callable
- The function you want to aggregate the flags with. It should be capable of operating on the flags dtype
- (usually ordered categorical).
- method : {'fagg', 'bagg', 'nagg'}, default 'nagg'
- Specifies which intervals to be aggregated for a certain timestamp. (preceeding, succeeding or
- "surrounding" interval). See description above for more details.
- to_drop : {List[str], str}, default None
- Flagtypes you want to drop before aggregation - effectively excluding values that are flagged
- with a flag in to_drop from the aggregation process. Default results in flagger.BAD
- values being dropped initially.
- """
- pass
-
-
-def linear(field, freq, to_drop):
- """
- A method to "regularize" data by interpolating linearly the data at regular timestamp.
-
- A series of data is considered "regular", if it is sampled regularly (= having uniform sampling rate).
-
- Interpolated values will get assigned the worst flag within freq-range.
-
- Note: the method will likely and significantly alter values and shape of ``data[field]``. The original data is kept
- in the data dios and assigned to the fieldname ``field + '_original'``.
-
- Note, that the data only gets interpolated at those (regular) timestamps, that have a valid (existing and
- not-na) datapoint preceeding them and one succeeding them within freq range.
- Regular timestamp that do not suffice this condition get nan assigned AND The associated flag will be of value
- ``flagger.UNFLAGGED``.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-regularized.
- freq : str
- An offset string. The frequency of the grid you want to interpolate your data at.
- to_drop : {List[str], str}, default None
- Flagtypes you want to drop before interpolation - effectively excluding values that are flagged
- with a flag in to_drop from the interpolation process. Default results in flagger.BAD
- values being dropped initially.
- """
- pass
-
-
-def interpolate(field, freq, method, order, to_drop):
- """
- A method to "regularize" data by interpolating the data at regular timestamp.
-
- A series of data is considered "regular", if it is sampled regularly (= having uniform sampling rate).
-
- Interpolated values will get assigned the worst flag within freq-range.
-
- There are available all the interpolations from the pandas.Series.interpolate method and they are called by
- the very same keywords.
-
- Note, that, to perform a timestamp aware, linear interpolation, you have to pass ``'time'`` as `method`,
- and NOT ``'linear'``.
-
- Note: the `method` will likely and significantly alter values and shape of ``data[field]``. The original data is
- kept in the data dios and assigned to the fieldname ``field + '_original'``.
-
- Note, that the data only gets interpolated at those (regular) timestamps, that have a valid (existing and
- not-na) datapoint preceeding them and one succeeding them within freq range.
- Regular timestamp that do not suffice this condition get nan assigned AND The associated flag will be of value
- ``flagger.UNFLAGGED``.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-regularized.
- freq : str
- An offset string. The frequency of the grid you want to interpolate your data at.
- method : {"linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric",
- "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"}: string
- The interpolation method you want to apply.
- order : int, default 1
- If your selected interpolation method can be performed at different *orders* - here you pass the desired
- order.
- to_drop : {List[str], str}, default None
- Flagtypes you want to drop before interpolation - effectively excluding values that are flagged
- with a flag in `to_drop` from the interpolation process. Default results in ``flagger.BAD``
- values being dropped initially.
- """
- pass
-
-
-def mapToOriginal(field, method, to_drop):
- """
- The Function function "undoes" regularization, by regaining the original data and projecting the
- flags calculated for the regularized data onto the original ones.
-
- Afterwards the regularized data is removed from the data dios and ``'field'`` will be associated
- with the original data "again".
-
- Wherever the flags in the original data are "better" then the regularized flags projected on them,
- they get overridden with this regularized flags value.
-
- Which regularized flags are to be projected on which original flags, is controlled by the "method" parameters.
-
- Generally, if you regularized with the method "X", you should pass the method "inverse_X" to the deharmonization.
- If you regularized with an interpolation, the method "inverse_interpolation" would be the appropriate choice.
- Also you should pass the same drop flags keyword.
-
- The deharm methods in detail:
- ("original_flags" are associated with the original data that is to be regained,
- "regularized_flags" are associated with the regularized data that is to be "deharmonized",
- "freq" refers to the regularized datas sampling frequencie)
-
- * ``'inverse_nagg'``: all original_flags within the range *+/- freq/2* of a regularized_flag, get assigned this
- regularized flags value. (if regularized_flags > original_flag)
- * ``'inverse_bagg'``: all original_flags succeeding a regularized_flag within the range of "freq", get assigned this
- regularized flags value. (if regularized_flag > original_flag)
- * ``'inverse_fagg'``: all original_flags preceeding a regularized_flag within the range of "freq", get assigned this
- regularized flags value. (if regularized_flag > original_flag)
-
- * ``'inverse_interpolation'``: all original_flags within the range *+/- freq* of a regularized_flag, get assigned this
- regularized flags value (if regularized_flag > original_flag).
-
- * ``'inverse_nshift'``: That original_flag within the range +/- *freq/2*, that is nearest to a regularized_flag,
- gets the regularized flags value. (if regularized_flag > original_flag)
- * ``'inverse_bshift'``: That original_flag succeeding a source flag within the range freq, that is nearest to a
- regularized_flag, gets assigned this regularized flags value. (if regularized_flag > original_flag)
- * ``'inverse_nshift'``: That original_flag preceeding a regularized flag within the range freq, that is nearest to a
- regularized_flag, gets assigned this regularized flags value. (if source_flag > original_flag)
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-deharmonized.
- method : {'inverse_fagg', 'inverse_bagg', 'inverse_nagg', 'inverse_fshift', 'inverse_bshift', 'inverse_nshift',
- 'inverse_interpolation'}
- The method used for projection of regularized flags onto original flags. See description above for more
- details.
- to_drop : {List[str], str}, default None
- Flagtypes you want to drop before interpolation - effectively excluding values that are flagged
- with a flag in to_drop from the interpolation process. Default results in flagger.BAD
- values being dropped initially.
- """
- pass
-
-
-def resample(field, freq, agg_func, max_invalid_total_d, max_invalid_consec_d, max_invalid_total_f, max_invalid_consec_f, flag_agg_func, empty_intervals_flag, to_drop, freq_check):
- """
- Function to resample the data. Afterwards the data will be sampled at regular (equidistant) timestamps
- (or Grid points). Sampling intervals therefor get aggregated with a function, specifyed by 'agg_func' parameter and
- the result gets projected onto the new timestamps with a method, specified by "method". The following method
- (keywords) are available:
-
- * ``'nagg'``: all values in the range (+/- `freq`/2) of a grid point get aggregated with agg_func and assigned to it.
- * ``'bagg'``: all values in a sampling interval get aggregated with agg_func and the result gets assigned to the last
- grid point.
- * ``'fagg'``: all values in a sampling interval get aggregated with agg_func and the result gets assigned to the next
- grid point.
-
-
- Note, that. if possible, functions passed to agg_func will get projected internally onto pandas.resample methods,
- wich results in some reasonable performance boost - however, for this to work, you should pass functions that have
- the __name__ attribute initialised and the according methods name assigned to it.
- Furthermore, you shouldnt pass numpys nan-functions
- (``nansum``, ``nanmean``,...) because those for example, have ``__name__ == 'nansum'`` and they will thus not
- trigger ``resample.func()``, but the slower ``resample.apply(nanfunc)``. Also, internally, no nans get passed to
- the functions anyway, so that there is no point in passing the nan functions.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-resampled.
- freq : str
- An Offset String, that will be interpreted as the frequency you want to resample your data with.
- agg_func : Callable
- The function you want to use for aggregation.
- method: {'fagg', 'bagg', 'nagg'}, default 'bagg'
- Specifies which intervals to be aggregated for a certain timestamp. (preceding, succeeding or
- "surrounding" interval). See description above for more details.
- max_invalid_total_d : {None, int}, default None
- Maximum number of invalid (nan) datapoints, allowed per resampling interval. If max_invalid_total_d is
- exceeded, the interval gets resampled to nan. By default (``np.inf``), there is no bound to the number of nan
- values in an interval and only intervals containing ONLY nan values or those, containing no values at all,
- get projected onto nan
- max_invalid_consec_d : {None, int}, default None
- Maximum number of consecutive invalid (nan) data points, allowed per resampling interval.
- If max_invalid_consec_d is exceeded, the interval gets resampled to nan. By default (np.inf),
- there is no bound to the number of consecutive nan values in an interval and only intervals
- containing ONLY nan values, or those containing no values at all, get projected onto nan.
- max_invalid_total_f : {None, int}, default None
- Same as `max_invalid_total_d`, only applying for the flags. The flag regarded as "invalid" value,
- is the one passed to empty_intervals_flag (default=``flagger.BAD``).
- Also this is the flag assigned to invalid/empty intervals.
- max_invalid_consec_f : {None, int}, default None
- Same as `max_invalid_total_f`, only applying onto flags. The flag regarded as "invalid" value, is the one passed
- to empty_intervals_flag (default=flagger.BAD). Also this is the flag assigned to invalid/empty intervals.
- flag_agg_func : Callable, default: max
- The function you want to aggregate the flags with. It should be capable of operating on the flags dtype
- (usually ordered categorical).
- empty_intervals_flag : {None, str}, default None
- A Flag, that you want to assign to invalid intervals. Invalid are those intervals, that contain nan values only,
- or no values at all. Furthermore the empty_intervals_flag is the flag, serving as "invalid" identifyer when
- checking for `max_total_invalid_f` and `max_consec_invalid_f patterns`. Default triggers ``flagger.BAD`` to be
- assigned.
- to_drop : {None, str, List[str]}, default None
- Flags that refer to values you want to drop before resampling - effectively excluding values that are flagged
- with a flag in to_drop from the resampling process - this means that they also will not be counted in the
- the `max_consec`/`max_total evaluation`. `to_drop` = ``None`` results in NO flags being dropped initially.
- freq_check : {None, 'check', 'auto'}, default None
-
- * ``None``: do not validate frequency-string passed to `freq`
- * ``'check'``: estimate frequency and log a warning if estimate miss matchs frequency string passed to 'freq', or
- if no uniform sampling rate could be estimated
- * ``'auto'``: estimate frequency and use estimate. (Ignores `freq` parameter.)
- """
- pass
-
-
-def reindexFlags(field, method, source, freq, to_drop, freq_check):
- """
- The Function projects flags of "source" onto flags of "field". Wherever the "field" flags are "better" then the
- source flags projected on them, they get overridden with this associated source flag value.
-
- Which "field"-flags are to be projected on which source flags, is controlled by the "method" and "freq"
- parameters.
-
- method: (field_flag in associated with "field", source_flags associated with "source")
-
- 'inverse_nagg' - all field_flags within the range +/- freq/2 of a source_flag, get assigned this source flags value.
- (if source_flag > field_flag)
- 'inverse_bagg' - all field_flags succeeding a source_flag within the range of "freq", get assigned this source flags
- value. (if source_flag > field_flag)
- 'inverse_fagg' - all field_flags preceeding a source_flag within the range of "freq", get assigned this source flags
- value. (if source_flag > field_flag)
-
- 'inverse_interpolation' - all field_flags within the range +/- freq of a source_flag, get assigned this source flags value.
- (if source_flag > field_flag)
-
- 'inverse_nshift' - That field_flag within the range +/- freq/2, that is nearest to a source_flag, gets the source
- flags value. (if source_flag > field_flag)
- 'inverse_bshift' - That field_flag succeeding a source flag within the range freq, that is nearest to a
- source_flag, gets assigned this source flags value. (if source_flag > field_flag)
- 'inverse_nshift' - That field_flag preceeding a source flag within the range freq, that is nearest to a
- source_flag, gets assigned this source flags value. (if source_flag > field_flag)
-
- 'match' - any field_flag with a timestamp matching a source_flags timestamp gets this source_flags value
- (if source_flag > field_flag)
-
- Note, to undo or backtrack a resampling/shifting/interpolation that has been performed with a certain method,
- you can just pass the associated "inverse" method. Also you should pass the same drop flags keyword.
-
- Parameters
- ----------
- field : str
- The fieldname of the data column, you want to project the source-flags onto.
- method : {'inverse_fagg', 'inverse_bagg', 'inverse_nagg', 'inverse_fshift', 'inverse_bshift', 'inverse_nshift'}
- The method used for projection of source flags onto field flags. See description above for more details.
- source : str
- The source source of flags projection.
- freq : {None, str},default None
- The freq determines the projection range for the projection method. See above description for more details.
- Defaultly (None), the sampling frequency of source is used.
- to_drop : {None, str, List[str]}, default None
- Flags referring to values that are to drop before flags projection. Relevant only when projecting with an
- inverted shift method. Defaultly flagger.BAD is listed.
- freq_check : {None, 'check', 'auto'}, default None
- - None: do not validate frequency-string passed to `freq`
- - 'check': estimate frequency and log a warning if estimate miss matchs frequency string passed to 'freq', or
- if no uniform sampling rate could be estimated
- - 'auto': estimate frequency and use estimate. (Ignores `freq` parameter.)
- """
- pass
-
diff --git a/docs/func_modules/residues.py b/docs/func_modules/residues.py
deleted file mode 100644
index ca6218621..000000000
--- a/docs/func_modules/residues.py
+++ /dev/null
@@ -1,65 +0,0 @@
-"""
-
-"""
-def calculatePolynomialResidues(field, winsz, polydeg, numba, eval_flags, min_periods):
- """
- Function fits a polynomial model to the data and returns the residues.
-
- The residue for value x is calculated by fitting a polynomial of degree "polydeg" to a data slice
- of size "winsz", wich has x at its center.
-
- Note, that the residues will be stored to the `field` field of the input data, so that the original data, the
- polynomial is fitted to, gets overridden.
-
- Note, that, if data[field] is not alligned to an equidistant frequency grid, the window size passed,
- has to be an offset string. Also numba boost options don`t apply for irregularly sampled
- timeseries.
-
- Note, that calculating the residues tends to be quite costy, because a function fitting is perfomed for every
- sample. To improve performance, consider the following possibillities:
-
- In case your data is sampled at an equidistant frequency grid:
-
- (1) If you know your data to have no significant number of missing values, or if you do not want to
- calculate residues for windows containing missing values any way, performance can be increased by setting
- min_periods=winsz.
-
- (2) If your data consists of more then around 200000 samples, setting numba=True, will boost the
- calculations up to a factor of 5 (for samplesize > 300000) - however for lower sample sizes,
- numba will slow down the calculations, also, up to a factor of 5, for sample_size < 50000.
- By default (numba='auto'), numba is set to true, if the data sample size exceeds 200000.
-
- in case your data is not sampled at an equidistant frequency grid:
-
- (1) Harmonization/resampling of your data will have a noticable impact on polyfittings performance - since
- numba_boost doesnt apply for irregularly sampled data in the current implementation.
-
- Note, that in the current implementation, the initial and final winsz/2 values do not get fitted.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-modelled.
- winsz : {str, int}
- The size of the window you want to use for fitting. If an integer is passed, the size
- refers to the number of periods for every fitting window. If an offset string is passed,
- the size refers to the total temporal extension. The window will be centered around the vaule-to-be-fitted.
- For regularly sampled timeseries the period number will be casted down to an odd number if
- even.
- polydeg : int
- The degree of the polynomial used for fitting
- numba : {True, False, "auto"}, default "auto"
- Wheather or not to apply numbas just-in-time compilation onto the poly fit function. This will noticably
- increase the speed of calculation, if the sample size is sufficiently high.
- If "auto" is selected, numba compatible fit functions get applied for data consisiting of > 200000 samples.
- eval_flags : bool, default True
- Wheather or not to assign new flags to the calculated residuals. If True, a residual gets assigned the worst
- flag present in the interval, the data for its calculation was obtained from.
- min_periods : {int, None}, default 0
- The minimum number of periods, that has to be available in every values fitting surrounding for the polynomial
- fit to be performed. If there are not enough values, np.nan gets assigned. Default (0) results in fitting
- regardless of the number of values present (results in overfitting for too sparse intervals). To automatically
- set the minimum number of periods to the number of values in an offset defined window size, pass np.nan.
- """
- pass
-
diff --git a/docs/func_modules/rolling.py b/docs/func_modules/rolling.py
deleted file mode 100644
index 9d06bcb80..000000000
--- a/docs/func_modules/rolling.py
+++ /dev/null
@@ -1,36 +0,0 @@
-"""
-
-"""
-def roll(field, winsz, func, eval_flags, min_periods, center):
- """
- Models the data with the rolling mean and returns the residues.
-
- Note, that the residues will be stored to the `field` field of the input data, so that the data that is modelled
- gets overridden.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-modelled.
- winsz : {int, str}
- The size of the window you want to roll with. If an integer is passed, the size
- refers to the number of periods for every fitting window. If an offset string is passed,
- the size refers to the total temporal extension.
- For regularly sampled timeseries, the period number will be casted down to an odd number if
- center = True.
- func : Callable[np.array, float], default np.mean
- Function to apply on the rolling window and obtain the curve fit value.
- eval_flags : bool, default True
- Wheather or not to assign new flags to the calculated residuals. If True, a residual gets assigned the worst
- flag present in the interval, the data for its calculation was obtained from.
- Currently not implemented in combination with not-harmonized timeseries.
- min_periods : int, default 0
- The minimum number of periods, that has to be available in every values fitting surrounding for the mean
- fitting to be performed. If there are not enough values, np.nan gets assigned. Default (0) results in fitting
- regardless of the number of values present.
- center : bool, default True
- Wheather or not to center the window the mean is calculated of around the reference value. If False,
- the reference value is placed to the right of the window (classic rolling mean with lag.)
- """
- pass
-
diff --git a/docs/func_modules/scores.py b/docs/func_modules/scores.py
deleted file mode 100644
index 765b667cf..000000000
--- a/docs/func_modules/scores.py
+++ /dev/null
@@ -1,81 +0,0 @@
-"""
-
-"""
-def assignKNNScore(field, n_neighbors, trafo, trafo_on_partition, scoring_func, target_field, partition_freq, partition_min, kNN_algorithm, metric, p, radius):
- """
- Score datapoints by an aggregation of the dictances to their k nearest neighbors.
-
- The function is a wrapper around the NearestNeighbors method from pythons sklearn library (See reference [1]).
-
- The steps taken to calculate the scores are as follows:
-
- 1. All the timeseries, named fields, are combined to one feature space by an *inner* join on their date time indexes.
- thus, only samples, that share timestamps across all fields will be included in the feature space
- 2. Any datapoint/sample, where one ore more of the features is invalid (=np.nan) will get excluded.
- 3. For every data point, the distance to its `n_neighbors` nearest neighbors is calculated by applying the
- metric `metric` at grade `p` onto the feature space. The defaults lead to the euclidian to be applied.
- If `radius` is not None, it sets the upper bound of distance for a neighbor to be considered one of the
- `n_neigbors` nearest neighbors. Furthermore, the `partition_freq` argument determines wich samples can be
- included into a datapoints nearest neighbors list, by segmenting the data into chunks of specified temporal
- extension and feeding that chunks to the kNN algorithm seperatly.
- 4. For every datapoint, the calculated nearest neighbors distances get aggregated to a score, by the function
- passed to `scoring_func`. The default, ``sum`` obviously just sums up the distances.
- 5. The resulting timeseries of scores gets assigned to the field target_field.
-
- Parameters
- ----------
- field : str
- The reference variable, the deviation from wich determines the flagging.
- n_neighbors : int, default 10
- The number of nearest neighbors to which the distance is comprised in every datapoints scoring calculation.
- trafo : Callable[np.array, np.array], default lambda x: x
- Transformation to apply on the variables before kNN scoring
- trafo_on_partition : bool, default True
- Weather or not to apply the transformation `trafo` onto the whole variable or onto each partition seperatly.
- scoring_func : Callable[numpy.array, float], default np.sum
- A function that assigns a score to every one dimensional array, containing the distances
- to every datapoints `n_neighbors` nearest neighbors.
- target_field : str, default 'kNN_scores'
- Name of the field, where the resulting scores should be written to.
- partition_freq : {np.inf, float, str}, default np.inf
- Determines the segmentation of the data into partitions, the kNN algorithm is
- applied onto individually.
-
- * ``np.inf``: Apply Scoring on whole data set at once
- * ``x`` > 0 : Apply scoring on successive data chunks of periods length ``x``
- * Offset String : Apply scoring on successive partitions of temporal extension matching the passed offset
- string
-
- partition_min : int, default 2
- The minimum number of periods that have to be present in a partition for the kNN scoring
- to be applied. If the number of periods present is below `partition_min`, the score for the
- datapoints in that partition will be np.nan.
- kNN_algorithm : {'ball_tree', 'kd_tree', 'brute', 'auto'}, default 'ball_tree'
- The search algorithm to find each datapoints k nearest neighbors.
- The keyword just gets passed on to the underlying sklearn method.
- See reference [1] for more information on the algorithm.
- metric : str, default 'minkowski'
- The metric the distances to any datapoints neighbors is computed with. The default of `metric`
- together with the default of `p` result in the euclidian to be applied.
- The keyword just gets passed on to the underlying sklearn method.
- See reference [1] for more information on the algorithm.
- p : int, default 2
- The grade of the metrice specified by parameter `metric`.
- The keyword just gets passed on to the underlying sklearn method.
- See reference [1] for more information on the algorithm.
- radius : {None, float}, default None
- If the radius is not None, only the distance to neighbors that ly within the range specified by `radius`
- are comprised in the scoring aggregation.
- The scoring method passed must be capable of handling np.nan values - since, for every point missing
- within `radius` range to make complete the list of the distances to the `n_neighbors` nearest neighbors,
- one np.nan value gets appended to the list passed to the scoring method.
- The keyword just gets passed on to the underlying sklearn method.
- See reference [1] for more information on the algorithm.
-
- References
- ----------
-
- [1] https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.NearestNeighbors.html
- """
- pass
-
diff --git a/docs/func_modules/tools.py b/docs/func_modules/tools.py
deleted file mode 100644
index 88578e724..000000000
--- a/docs/func_modules/tools.py
+++ /dev/null
@@ -1,128 +0,0 @@
-"""
-
-"""
-def copy(field):
- """
- The function generates a copy of the data "field" and inserts it under the name field + suffix into the existing
- data.
-
- Parameters
- ----------
- field : str
- The fieldname of the data column, you want to fork (copy).
- """
- pass
-
-
-def drop(field):
- """
- The function drops field from the data dios and the flagger.
-
- Parameters
- ----------
- field : str
- The fieldname of the data column, you want to drop.
- """
- pass
-
-
-def rename(field, new_name):
- """
- The function renames field to new name (in both, the flagger and the data).
-
- Parameters
- ----------
- field : str
- The fieldname of the data column, you want to rename.
- new_name : str
- String, field is to be replaced with.
- """
- pass
-
-
-def mask(field, mode, mask_var, period_start, period_end, include_bounds):
- """
- This function realizes masking within saqc.
-
- Due to some inner saqc mechanics, it is not straight forwardly possible to exclude
- values or datachunks from flagging routines. This function replaces flags with np.nan
- value, wherever values are to get masked. Furthermore, the masked values get replaced by
- np.nan, so that they dont effect calculations.
-
- Here comes a recipe on how to apply a flagging function only on a masked chunk of the variable field:
-
- 1. dublicate "field" in the input data (proc_copy)
- 2. mask the dublicated data (modelling_mask)
- 3. apply the tests you only want to be applied onto the masked data chunks (saqc_tests)
- 4. project the flags, calculated on the dublicated and masked data onto the original field data
- (proc_projectFlags or flagGeneric)
- 5. drop the dublicated data (proc_drop)
-
- To see an implemented example, checkout flagSeasonalRange in the saqc.functions module
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-masked.
- mode : {"periodic", "mask_var"}
- The masking mode.
- - "periodic": parameters "period_start", "period_end" are evaluated to generate a periodical mask
- - "mask_var": data[mask_var] is expected to be a boolean valued timeseries and is used as mask.
- mask_var : {None, str}, default None
- Only effective if mode == "mask_var"
- Fieldname of the column, holding the data that is to be used as mask. (must be moolean series)
- Neither the series` length nor its labels have to match data[field]`s index and length. An inner join of the
- indices will be calculated and values get masked where the values of the inner join are "True".
- period_start : {None, str}, default None
- Only effective if mode == "seasonal"
- String denoting starting point of every period. Formally, it has to be a truncated instance of "mm-ddTHH:MM:SS".
- Has to be of same length as `period_end` parameter.
- See examples section below for some examples.
- period_end : {None, str}, default None
- Only effective if mode == "periodic"
- String denoting starting point of every period. Formally, it has to be a truncated instance of "mm-ddTHH:MM:SS".
- Has to be of same length as `period_end` parameter.
- See examples section below for some examples.
- include_bounds : boolean
- Wheather or not to include the mask defining bounds to the mask.
-
- Examples
- --------
- The `period_start` and `period_end` parameters provide a conveniant way to generate seasonal / date-periodic masks.
- They have to be strings of the forms: "mm-ddTHH:MM:SS", "ddTHH:MM:SS" , "HH:MM:SS", "MM:SS" or "SS"
- (mm=month, dd=day, HH=hour, MM=minute, SS=second)
- Single digit specifications have to be given with leading zeros.
- `period_start` and `seas on_end` strings have to be of same length (refer to the same periodicity)
- The highest date unit gives the period.
- For example:
-
- >>> period_start = "01T15:00:00"
- >>> period_end = "13T17:30:00"
-
- Will result in all values sampled between 15:00 at the first and 17:30 at the 13th of every month get masked
-
- >>> period_start = "01:00"
- >>> period_end = "04:00"
-
- All the values between the first and 4th minute of every hour get masked.
-
- >>> period_start = "01-01T00:00:00"
- >>> period_end = "01-03T00:00:00"
-
- Mask january and february of evcomprosed in theery year. masking is inclusive always, so in this case the mask will
- include 00:00:00 at the first of march. To exclude this one, pass:
-
- >>> period_start = "01-01T00:00:00"
- >>> period_end = "02-28T23:59:59"
-
- To mask intervals that lap over a seasons frame, like nights, or winter, exchange sequence of season start and
- season end. For example, to mask night hours between 22:00:00 in the evening and 06:00:00 in the morning, pass:
-
- >>> period_start = "22:00:00"
- >>> period_end = "06:00:00"
-
- When inclusive_selection="season", all above examples work the same way, only that you now
- determine wich values NOT TO mask (=wich values are to constitute the "seasons").
- """
- pass
-
diff --git a/docs/func_modules/transformation.py b/docs/func_modules/transformation.py
deleted file mode 100644
index 8ee5a5bae..000000000
--- a/docs/func_modules/transformation.py
+++ /dev/null
@@ -1,25 +0,0 @@
-"""
-
-"""
-def transform(field, func, partition_freq):
- """
- Function to transform data columns with a transformation that maps series onto series of the same length.
-
- Note, that flags get preserved.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-transformed.
- func : Callable[{pd.Series, np.array}, np.array]
- Function to transform data[field] with.
- partition_freq : {None, float, str}, default None
- Determines the segmentation of the data into partitions, the transformation is applied on individually
-
- * ``np.inf``: Apply transformation on whole data set at once
- * ``x`` > 0 : Apply transformation on successive data chunks of periods length ``x``
- * Offset String : Apply transformation on successive partitions of temporal extension matching the passed offset
- string
- """
- pass
-
diff --git a/docs/intro_modules/AdvancedFlagging.py b/docs/intro_modules/AdvancedFlagging.py
deleted file mode 100644
index 22d84a7e0..000000000
--- a/docs/intro_modules/AdvancedFlagging.py
+++ /dev/null
@@ -1,66 +0,0 @@
-"""
-
-"""
-def flagPatternByDTW(field):
- """
- Pattern recognition via wavelets.
-
- The steps are:
- 1. work on chunks returned by a moving window
- 2. each chunk is compared to the given pattern, using the wavelet algorithm as presented in [1]
- 3. if the compared chunk is equal to the given pattern it gets flagged
-
- Parameters
- ----------
-
- field : str
- The fieldname of the data column, you want to correct.
- """
- pass
-
-
-def flagOffset(field, thresh, tolerance, window, numba_kickin):
- """
- A basic outlier test that is designed to work for harmonized and not harmonized data.
-
- The test classifies values/value courses as outliers by detecting not only a rise in value, but also,
- checking for a return to the initial value level.
-
- Values :math:`x_n, x_{n+1}, .... , x_{n+k}` of a timeseries :math:`x` with associated timestamps
- :math:`t_n, t_{n+1}, .... , t_{n+k}` are considered spikes, if
-
- 1. :math:`|x_{n-1} - x_{n + s}| >` `thresh`, for all :math:`s \in [0,1,2,...,k]`
-
- 2. :math:`|x_{n-1} - x_{n+k+1}| <` `tolerance`
-
- 3. :math:`|t_{n-1} - t_{n+k+1}| <` `window`
-
- Note, that this definition of a "spike" not only includes one-value outliers, but also plateau-ish value courses.
-
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged. (Here a dummy, for structural reasons)
- thresh : float
- Minimum difference between to values, to consider the latter one as a spike. See condition (1)
- tolerance : float
- Maximum difference between pre-spike and post-spike values. See condition (2)
- window : {str, int}, default '15min'
- Maximum length of "spiky" value courses. See condition (3). Integer defined window length are only allowed for
- regularly sampled timeseries.
- numba_kickin : int, default 200000
- When there are detected more than `numba_kickin` incidents of potential spikes,
- the pandas.rolling - part of computation gets "jitted" with numba.
- Default value hast proven to be around the break even point between "jit-boost" and "jit-costs".
-
-
- References
- ----------
- The implementation is a time-window based version of an outlier test from the UFZ Python library,
- that can be found here:
-
- https://git.ufz.de/chs/python/blob/master/ufz/level1/spike.py
- """
- pass
-
diff --git a/docs/intro_modules/BasicFlagging.py b/docs/intro_modules/BasicFlagging.py
deleted file mode 100644
index 0032a5c8f..000000000
--- a/docs/intro_modules/BasicFlagging.py
+++ /dev/null
@@ -1,32 +0,0 @@
-"""
-
-"""
-def flagRange(field, min, max):
- """
- Function flags values not covered by the closed interval [`min`, `max`].
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged.
- min : float
- Lower bound for valid data.
- max : float
- Upper bound for valid data.
- """
- pass
-
-
-def flagMissing(field, nodata):
- """
- The function flags all values indicating missing data.
-
- Parameters
- ----------
- field : str
- The fieldname of the column, holding the data-to-be-flagged.
- nodata : any, default np.nan
- A value that defines missing data.
- """
- pass
-
diff --git a/docs/intro_modules/module_dict.pkl b/docs/intro_modules/module_dict.pkl
deleted file mode 100644
index 23f577445c0f9414ab46ee4d171ebaeba97edd8d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001
literal 238
zcmZo*t}SHHh~Q;lU~oz-&P;a8NlZ`A%u6q1j4fo+h!6sb=a-h`WTqAs>!ktZf)ex6
zQwy2F5=lj=iP;cg-^}7-kS><g2r+b{;!~1~ONv0UtPwythC;Ro0U+g=QkIyPoSFi+
zkv+DMLnA^0C{>VHQj%Jf2evQ(!gi{32?;Ocj1UG&qd3SvEv-1Uq>w8$LK2Ii2)ns6
H*h=*Pt?Ei|
--
GitLab
From baf2063d654380d437d55331621661f6215c6240 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Tue, 23 Mar 2021 23:39:18 +0100
Subject: [PATCH 080/180] exposed 'flag' to signature
---
saqc/core/modules/breaks.py | 4 ++
saqc/core/modules/changepoints.py | 3 ++
saqc/core/modules/constants.py | 11 ++++--
saqc/core/modules/curvefit.py | 21 ++++++----
saqc/core/modules/drift.py | 5 +++
saqc/core/modules/flagtools.py | 3 +-
saqc/core/modules/generic.py | 19 +++++++--
saqc/core/modules/outliers.py | 15 +++++++-
saqc/core/modules/pattern.py | 11 ++++--
saqc/core/modules/resampling.py | 2 +
saqc/core/modules/residues.py | 3 ++
saqc/core/modules/rolling.py | 2 +
saqc/funcs/breaks.py | 16 ++++++--
saqc/funcs/changepoints.py | 38 ++++++++++++------
saqc/funcs/constants.py | 19 ++++++---
saqc/funcs/curvefit.py | 40 +++++++++++--------
saqc/funcs/drift.py | 47 +++++++++++++++--------
saqc/funcs/flagtools.py | 12 ++++--
saqc/funcs/generic.py | 24 +++++++++---
saqc/funcs/outliers.py | 64 ++++++++++++++++++++++++-------
saqc/funcs/pattern.py | 22 ++++++-----
saqc/funcs/resampling.py | 15 +++++++-
saqc/funcs/residues.py | 7 ++++
saqc/funcs/rolling.py | 9 ++---
24 files changed, 299 insertions(+), 113 deletions(-)
diff --git a/saqc/core/modules/breaks.py b/saqc/core/modules/breaks.py
index 02b237a60..b07edba09 100644
--- a/saqc/core/modules/breaks.py
+++ b/saqc/core/modules/breaks.py
@@ -5,6 +5,7 @@ from typing import Tuple
import numpy as np
from dios import DictOfSeries
+from saqc.constants import *
from saqc import Flagger
from saqc.core.modules.base import ModuleBase
from saqc.lib.types import FreqString, IntegerWindow, ColumnName
@@ -16,6 +17,7 @@ class Breaks(ModuleBase):
self,
field: ColumnName,
nodata: float = np.nan,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagMissing", locals())
@@ -25,6 +27,7 @@ class Breaks(ModuleBase):
field: ColumnName,
gap_window: FreqString,
group_window: FreqString,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagIsolated", locals())
@@ -35,6 +38,7 @@ class Breaks(ModuleBase):
thresh: float,
winsz: FreqString,
min_periods: IntegerWindow = 1,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagJumps", locals())
diff --git a/saqc/core/modules/changepoints.py b/saqc/core/modules/changepoints.py
index bab02fc86..19ed26d29 100644
--- a/saqc/core/modules/changepoints.py
+++ b/saqc/core/modules/changepoints.py
@@ -7,6 +7,7 @@ import numpy as np
from dios import DictOfSeries
from typing_extensions import Literal
+from saqc.constants import *
from saqc import Flagger
from saqc.core.modules.base import ModuleBase
from saqc.lib.types import FreqString, IntegerWindow
@@ -26,6 +27,7 @@ class ChangePoints(ModuleBase):
try_to_jit: bool = True, # TODO rm, not a user decision
reduce_window: FreqString = None,
reduce_func: Callable[[np.ndarray, np.ndarray], int] = lambda x, _: x.argmax(),
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagChangePoints", locals())
@@ -45,6 +47,7 @@ class ChangePoints(ModuleBase):
model_by_resids: bool = False,
flag_changepoints: bool = False,
assign_cluster: bool = True,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("assignChangePointCluster", locals())
diff --git a/saqc/core/modules/constants.py b/saqc/core/modules/constants.py
index 09f55eb00..22239aa09 100644
--- a/saqc/core/modules/constants.py
+++ b/saqc/core/modules/constants.py
@@ -4,6 +4,7 @@ from typing import Tuple
from dios import DictOfSeries
+from saqc.constants import *
from saqc import Flagger
from saqc.core.modules.base import ModuleBase
from saqc.lib.types import FreqString, ColumnName
@@ -14,10 +15,11 @@ class Constants(ModuleBase):
def flagByVariance(
self,
field: ColumnName,
- window: FreqString="12h",
- thresh: float=0.0005,
- max_missing: int=None,
- max_consec_missing: int=None,
+ window: FreqString = "12h",
+ thresh: float = 0.0005,
+ max_missing: int = None,
+ max_consec_missing: int = None,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagByVariance", locals())
@@ -27,6 +29,7 @@ class Constants(ModuleBase):
field: ColumnName,
thresh: float,
window: FreqString,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagConstants", locals())
diff --git a/saqc/core/modules/curvefit.py b/saqc/core/modules/curvefit.py
index 595126406..c24ce08b0 100644
--- a/saqc/core/modules/curvefit.py
+++ b/saqc/core/modules/curvefit.py
@@ -5,17 +5,22 @@ from typing import Union, Tuple
from dios import DictOfSeries
from typing_extensions import Literal
+from saqc.constants import *
from saqc import Flagger
from saqc.core.modules.base import ModuleBase
class Curvefit(ModuleBase):
- def fitPolynomial(self, field: str,
- winsz: Union[int, str],
- polydeg: int,
- numba: Literal[True, False, "auto"] = "auto",
- eval_flags: bool = True,
- min_periods: int = 0,
- return_residues: bool = False,
- **kwargs) -> Tuple[DictOfSeries, Flagger]:
+ def fitPolynomial(
+ self,
+ field: str,
+ winsz: Union[int, str],
+ polydeg: int,
+ numba: Literal[True, False, "auto"] = "auto",
+ eval_flags: bool = True,
+ min_periods: int = 0,
+ return_residues: bool = False,
+ flag: float = BAD,
+ **kwargs
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("fitPolynomial", locals())
diff --git a/saqc/core/modules/drift.py b/saqc/core/modules/drift.py
index 3c422fc93..e063e62f3 100644
--- a/saqc/core/modules/drift.py
+++ b/saqc/core/modules/drift.py
@@ -6,6 +6,7 @@ from typing import Sequence, Callable, Optional, Tuple
import numpy as np
from scipy.spatial.distance import pdist
+from saqc.constants import *
from saqc.core.modules.base import ModuleBase
from saqc.funcs import LinkageString, DictOfSeries, Flagger
from saqc.lib.types import ColumnName, FreqString, CurveFitter
@@ -21,6 +22,7 @@ class Drift(ModuleBase):
norm_frac: float = 0.5,
metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
linkage_method: LinkageString = "single",
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagDriftFromNorm", locals())
@@ -32,6 +34,7 @@ class Drift(ModuleBase):
segment_freq: FreqString,
thresh: float,
metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagDriftFromReference", locals())
@@ -46,6 +49,7 @@ class Drift(ModuleBase):
norm_frac: float = 0.5,
metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
linkage_method: LinkageString = "single",
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagDriftFromScaledNorm", locals())
@@ -56,6 +60,7 @@ class Drift(ModuleBase):
maint_data_field: ColumnName,
cal_mean: int = 5,
flag_maint_period: bool = False,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("correctExponentialDrift", locals())
diff --git a/saqc/core/modules/flagtools.py b/saqc/core/modules/flagtools.py
index 426dfb276..7cc2b1633 100644
--- a/saqc/core/modules/flagtools.py
+++ b/saqc/core/modules/flagtools.py
@@ -41,7 +41,8 @@ class FlagTools(ModuleBase):
self, field: ColumnName,
mdata: Union[pd.Series, pd.DataFrame, DictOfSeries],
mflag: Any = 1,
- method=Literal["plain", "ontime", "left-open", "right-open"],
+ method: Literal["plain", "ontime", "left-open", "right-open"] = 'plain',
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagManual", locals())
diff --git a/saqc/core/modules/generic.py b/saqc/core/modules/generic.py
index 3f44c45f7..da80700c3 100644
--- a/saqc/core/modules/generic.py
+++ b/saqc/core/modules/generic.py
@@ -13,10 +13,21 @@ from saqc.core.modules.base import ModuleBase
class Generic(ModuleBase):
- def process(self, field: str, func: Callable[[pd.Series], pd.Series],
- nodata: float = np.nan, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+ def process(
+ self,
+ field: str,
+ func: Callable[[pd.Series], pd.Series],
+ nodata: float = np.nan,
+ **kwargs
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("process", locals())
- def flag(self, field: str, func: Callable[[pd.Series], pd.Series],
- nodata: float = np.nan, flag=BAD, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+ def flag(
+ self,
+ field: str,
+ func: Callable[[pd.Series], pd.Series],
+ nodata: float = np.nan,
+ flag: float = BAD,
+ **kwargs
+ ) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flag", locals())
diff --git a/saqc/core/modules/outliers.py b/saqc/core/modules/outliers.py
index 08efb1068..bf8152039 100644
--- a/saqc/core/modules/outliers.py
+++ b/saqc/core/modules/outliers.py
@@ -8,6 +8,7 @@ import pandas as pd
from dios import DictOfSeries
from typing_extensions import Literal
+from saqc.constants import *
from saqc import Flagger
from saqc.core.modules.base import ModuleBase
from saqc.lib.types import IntegerWindow, FreqString, ColumnName
@@ -22,6 +23,7 @@ class Outliers(ModuleBase):
partition_min: int = 11,
iter_start: float = 0.5,
alpha: float = 0.05,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagByStray", locals())
@@ -42,6 +44,7 @@ class Outliers(ModuleBase):
reduction_drop_flagged: bool = False, # TODO: still a case ?
reduction_thresh: float = 3.5,
reduction_min_periods: int = 1,
+ flag: float = BAD,
**kwargs,
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagMVScores", locals())
@@ -57,12 +60,18 @@ class Outliers(ModuleBase):
min_slope: Optional[float] = None,
min_slope_weight: float = 0.8,
numba_boost: bool = True, # TODO: rm, not a user decision
+ flag: float = BAD,
**kwargs,
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagRaise", locals())
def flagMAD(
- self, field: ColumnName, window: FreqString, z: float = 3.5, **kwargs
+ self,
+ field: ColumnName,
+ window: FreqString,
+ z: float = 3.5,
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagMAD", locals())
@@ -74,6 +83,7 @@ class Outliers(ModuleBase):
window: Union[IntegerWindow, FreqString],
rel_thresh: Optional[float] = None,
numba_kickin: int = 200000, # TODO: rm, not a user decision
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagOffset", locals())
@@ -85,6 +95,7 @@ class Outliers(ModuleBase):
alpha: float = 0.05,
min_periods: int = 8,
check_lagged: bool = False,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagByGrubbs", locals())
@@ -94,6 +105,7 @@ class Outliers(ModuleBase):
field: ColumnName,
min: float = -np.inf,
max: float = np.inf,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagRange", locals())
@@ -104,6 +116,7 @@ class Outliers(ModuleBase):
fields: Sequence[ColumnName],
thresh: float,
cross_stat: Literal["modZscore", "Zscore"] = "modZscore",
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagCrossStatistic", locals())
diff --git a/saqc/core/modules/pattern.py b/saqc/core/modules/pattern.py
index 06c9ab26c..38d083945 100644
--- a/saqc/core/modules/pattern.py
+++ b/saqc/core/modules/pattern.py
@@ -5,6 +5,7 @@ from typing import Sequence, Tuple
from dios import DictOfSeries
+from saqc.constants import *
from saqc import Flagger
from saqc.core.modules.base import ModuleBase
@@ -15,8 +16,9 @@ class Pattern(ModuleBase):
self,
field: str,
ref_field: str,
- widths: Sequence[int]=(1, 2, 4, 8),
- waveform: str="mexh",
+ widths: Sequence[int] = (1, 2, 4, 8),
+ waveform: str = "mexh",
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagPatternByDTW", locals())
@@ -25,8 +27,9 @@ class Pattern(ModuleBase):
self,
field: str,
ref_field: str,
- max_distance: float=0.03,
- normalize: bool=True,
+ max_distance: float = 0.03,
+ normalize: bool = True,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("flagPatternByWavelet", locals())
diff --git a/saqc/core/modules/resampling.py b/saqc/core/modules/resampling.py
index be4859bb5..9822bbd8b 100644
--- a/saqc/core/modules/resampling.py
+++ b/saqc/core/modules/resampling.py
@@ -8,6 +8,7 @@ import pandas as pd
from dios import DictOfSeries
from typing_extensions import Literal
+from saqc.constants import *
from saqc import Flagger
from saqc.core.modules.base import ModuleBase
from saqc.funcs.interpolation import _SUPPORTED_METHODS
@@ -22,6 +23,7 @@ class Resampling(ModuleBase):
value_func,
flag_func: Callable[[pd.Series], float] = np.nanmax,
method: Literal["fagg", "bagg", "nagg"] = "nagg",
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("aggregate", locals())
diff --git a/saqc/core/modules/residues.py b/saqc/core/modules/residues.py
index 85d7426f0..877323546 100644
--- a/saqc/core/modules/residues.py
+++ b/saqc/core/modules/residues.py
@@ -7,6 +7,7 @@ import numpy as np
from dios import DictOfSeries
from typing_extensions import Literal
+from saqc.constants import *
from saqc import Flagger
from saqc.core.modules.base import ModuleBase
@@ -21,6 +22,7 @@ class Residues(ModuleBase):
numba: Literal[True, False, "auto"] = "auto", # TODO: rm, not a a user decision
eval_flags: bool = True, # TODO, not valid anymore, if still needed, maybe assign user-passed ``flag``?
min_periods: Optional[int] = 0,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("calculatePolynomialResidues", locals())
@@ -33,6 +35,7 @@ class Residues(ModuleBase):
eval_flags: bool = True,
min_periods: Optional[int] = 0,
center: bool = True,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
return self.defer("calculateRollingResidues", locals())
diff --git a/saqc/core/modules/rolling.py b/saqc/core/modules/rolling.py
index f9c6be163..d29cb4018 100644
--- a/saqc/core/modules/rolling.py
+++ b/saqc/core/modules/rolling.py
@@ -6,6 +6,7 @@ from typing import Union, Callable
import numpy as np
import pandas as pd
+from saqc.constants import *
from saqc.core.modules.base import ModuleBase
@@ -19,6 +20,7 @@ class Rolling(ModuleBase):
min_periods: int=0,
center: bool=True,
return_residues=False, # TODO: this should not be public, a wrapper would be better
+ flag: float = BAD,
**kwargs
):
return self.defer("roll", locals())
diff --git a/saqc/funcs/breaks.py b/saqc/funcs/breaks.py
index 6c394e3e7..8f10e9b72 100644
--- a/saqc/funcs/breaks.py
+++ b/saqc/funcs/breaks.py
@@ -16,6 +16,7 @@ import pandas.tseries.frequencies
from dios import DictOfSeries
+from saqc.constants import *
from saqc.lib.tools import groupConsecutives
from saqc.lib.types import FreqString, ColumnName, IntegerWindow
from saqc.funcs.changepoints import assignChangePointCluster
@@ -29,6 +30,7 @@ def flagMissing(
field: ColumnName,
flagger: Flagger,
nodata: float = np.nan,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -44,6 +46,8 @@ def flagMissing(
A flagger object, holding flags and additional Informations related to `data`.
nodata : any, default np.nan
A value that defines missing data.
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -59,7 +63,7 @@ def flagMissing(
else:
mask = datacol == nodata
- flagger[mask, field] = kwargs['flag']
+ flagger[mask, field] = flag
return data, flagger
@@ -70,6 +74,7 @@ def flagIsolated(
flagger: Flagger,
gap_window: FreqString,
group_window: FreqString,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -92,6 +97,8 @@ def flagIsolated(
group_window : str
The maximum temporal extension allowed for a group that is isolated by gaps of size 'gap_window',
to be actually flagged as isolated group. See condition (1).
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -130,7 +137,7 @@ def flagIsolated(
if right.all():
flags[start:stop] = True
- flagger[mask, field] = kwargs['flag']
+ flagger[mask, field] = flag
return data, flagger
@@ -142,6 +149,7 @@ def flagJumps(
thresh: float,
winsz: FreqString,
min_periods: IntegerWindow = 1,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -163,6 +171,8 @@ def flagJumps(
min_periods : int, default 1
Minimum number of periods that have to be present in a window of size `winsz`, so that
the mean value obtained from that window is regarded valid.
+ flag : float, default BAD
+ flag to set.
"""
return assignChangePointCluster(
data, field, flagger,
@@ -173,6 +183,6 @@ def flagJumps(
flag_changepoints=True,
model_by_resids=False,
assign_cluster=False,
+ flag=flag,
**kwargs
)
-
diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py
index 7025ad712..4ef620f54 100644
--- a/saqc/funcs/changepoints.py
+++ b/saqc/funcs/changepoints.py
@@ -33,6 +33,7 @@ def flagChangePoints(
try_to_jit: bool = True, # TODO rm, not a user decision
reduce_window: FreqString = None,
reduce_func: Callable[[np.ndarray, np.ndarray], int] = lambda x, _: x.argmax(),
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -79,19 +80,31 @@ def flagChangePoints(
First input parameter will hold the result from the stat_func evaluation for every
reduction window. Second input parameter holds the result from the thresh_func evaluation.
The default reduction function just selects the value that maximizes the stat_func.
-
+ flag : float, default BAD
+ flag to set.
Returns
-------
-
"""
return assignChangePointCluster(
- data, field, flagger, stat_func=stat_func, thresh_func=thresh_func,
- bwd_window=bwd_window, min_periods_bwd=min_periods_bwd,
- fwd_window=fwd_window, min_periods_fwd=min_periods_fwd, closed=closed,
- try_to_jit=try_to_jit, reduce_window=reduce_window,
- reduce_func=reduce_func, flag_changepoints=True, model_by_resids=False,
- assign_cluster=False, **kwargs
+ data,
+ field,
+ flagger,
+ stat_func=stat_func,
+ thresh_func=thresh_func,
+ bwd_window=bwd_window,
+ min_periods_bwd=min_periods_bwd,
+ fwd_window=fwd_window,
+ min_periods_fwd=min_periods_fwd,
+ closed=closed,
+ try_to_jit=try_to_jit,
+ reduce_window=reduce_window,
+ reduce_func=reduce_func,
+ flag_changepoints=True,
+ model_by_resids=False,
+ assign_cluster=False,
+ flag=flag,
+ **kwargs
)
@@ -111,6 +124,7 @@ def assignChangePointCluster(
model_by_resids: bool = False,
flag_changepoints: bool = False,
assign_cluster: bool = True,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -160,15 +174,16 @@ def assignChangePointCluster(
reduction window. Second input parameter holds the result from the thresh_func evaluation.
The default reduction function just selects the value that maximizes the stat_func.
flag_changepoints : bool, default False
- If true, the points, where there is a change in data modelling regime detected get flagged BAD.
+ If true, the points, where there is a change in data modelling regime detected gets flagged.
model_by_resids : bool, default False
If True, the data is replaced by the stat_funcs results instead of regime labels.
assign_cluster : bool, default True
Is set to False, if called by function that oly wants to calculate flags.
+ flag : float, default BAD
+ flag to set.
Returns
-------
-
"""
data = data.copy()
data_ser = data[field].dropna()
@@ -242,8 +257,7 @@ def assignChangePointCluster(
flagger[:, field] = UNFLAGGED
if flag_changepoints:
- # TODO: does not respect kwargs[flag]
- flagger[det_index, field] = BAD
+ flagger[det_index, field] = flag
return data, flagger
diff --git a/saqc/funcs/constants.py b/saqc/funcs/constants.py
index 02327498f..5d0b30804 100644
--- a/saqc/funcs/constants.py
+++ b/saqc/funcs/constants.py
@@ -9,6 +9,7 @@ import pandas as pd
from dios import DictOfSeries
+from saqc.constants import *
from saqc.core.register import register
from saqc.flagger import Flagger
from saqc.lib.ts_operators import varQC
@@ -23,6 +24,7 @@ def flagConstants(
flagger: Flagger,
thresh: float,
window: FreqString,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -48,6 +50,8 @@ def flagConstants(
Upper bound for the maximum total change of an interval to be flagged constant.
window : str
Lower bound for the size of an interval to be flagged constant.
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -73,7 +77,7 @@ def flagConstants(
m2 = r.max() - r.min() <= thresh
mask = m1 | m2
- flagger[mask, field] = kwargs['flag']
+ flagger[mask, field] = flag
return data, flagger
@@ -82,10 +86,11 @@ def flagByVariance(
data: DictOfSeries,
field: ColumnName,
flagger: Flagger,
- window: FreqString="12h",
- thresh: float=0.0005,
- max_missing: int=None,
- max_consec_missing: int=None,
+ window: FreqString = "12h",
+ thresh: float = 0.0005,
+ max_missing: int = None,
+ max_consec_missing: int = None,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -114,6 +119,8 @@ def flagByVariance(
Maximum number of consecutive nan values allowed in an interval to retrieve a
valid variance from it. (Intervals with a number of nans exceeding
"max_consec_missing" have no chance to get flagged a plateau!)
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -154,5 +161,5 @@ def flagByVariance(
# result:
plateaus = (plateaus[plateaus == 1.0]).index
- flagger[plateaus, field] = kwargs['flag']
+ flagger[plateaus, field] = flag
return data, flagger
diff --git a/saqc/funcs/curvefit.py b/saqc/funcs/curvefit.py
index fc04cbeac..f77b75346 100644
--- a/saqc/funcs/curvefit.py
+++ b/saqc/funcs/curvefit.py
@@ -4,29 +4,37 @@
from math import floor
from typing import Tuple, Union
from typing_extensions import Literal
-
import numpy as np
import pandas as pd
-
from dios import DictOfSeries
+from saqc.constants import *
from saqc.core.register import register
-
from saqc.lib.tools import getFreqDelta
from saqc.flagger import Flagger
-from saqc.lib.ts_operators import polyRollerIrregular, polyRollerNumba, polyRoller, polyRollerNoMissingNumba, \
+from saqc.lib.ts_operators import (
+ polyRollerIrregular,
+ polyRollerNumba,
+ polyRoller,
+ polyRollerNoMissingNumba,
polyRollerNoMissing
+)
@register(masking='field', module="curvefit")
-def fitPolynomial(data: DictOfSeries, field: str, flagger: Flagger,
- winsz: Union[int, str],
- polydeg: int,
- numba: Literal[True, False, "auto"] = "auto",
- eval_flags: bool = True,
- min_periods: int = 0,
- return_residues: bool = False,
- **kwargs) -> Tuple[DictOfSeries, Flagger]:
+def fitPolynomial(
+ data: DictOfSeries,
+ field: str,
+ flagger: Flagger,
+ winsz: Union[int, str],
+ polydeg: int,
+ numba: Literal[True, False, "auto"] = "auto",
+ eval_flags: bool = True,
+ min_periods: int = 0,
+ return_residues: bool = False,
+ flag: float = BAD,
+ **kwargs
+) -> Tuple[DictOfSeries, Flagger]:
"""
Function fits a polynomial model to the data and returns the fitted data curve.
@@ -91,6 +99,8 @@ def fitPolynomial(data: DictOfSeries, field: str, flagger: Flagger,
set the minimum number of periods to the number of values in an offset defined window size, pass np.nan.
return_residues : bool, default False
Internal parameter. Makes the method return the residues instead of the fit.
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -149,8 +159,8 @@ def fitPolynomial(data: DictOfSeries, field: str, flagger: Flagger,
lambda x, y: x[y], raw=True, args=(center_index,)
)
- # we need a missing value marker that is not nan, because nan values dont get passed by pandas rolling
- # method
+ # we need a missing value marker that is not nan,
+ # because nan values dont get passed by pandas rolling method
miss_marker = to_fit.min()
miss_marker = np.floor(miss_marker - 1)
na_mask = to_fit.isna()
@@ -192,8 +202,6 @@ def fitPolynomial(data: DictOfSeries, field: str, flagger: Flagger,
data[field] = residues
if eval_flags:
- # with the new flagger we dont have to care
- # about to set NaNs to the original flags anymore
# TODO: we does not get any flags here, because of masking=field
worst = flagger[field].rolling(winsz, center=True, min_periods=min_periods).max()
flagger[field] = worst
diff --git a/saqc/funcs/drift.py b/saqc/funcs/drift.py
index 14417c3f0..d8605f67d 100644
--- a/saqc/funcs/drift.py
+++ b/saqc/funcs/drift.py
@@ -1,17 +1,19 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
+
+import numpy as np
+import pandas as pd
import functools
+from dios import DictOfSeries
+
from typing import Optional, Tuple, Sequence, Callable, Optional
from typing_extensions import Literal
-import numpy as np
-import pandas as pd
from scipy import stats
from scipy.optimize import curve_fit
from scipy.spatial.distance import pdist
-from dios import DictOfSeries
-
+from saqc.constants import *
from saqc.core.register import register
from saqc.flagger import Flagger
from saqc.funcs.resampling import shift
@@ -35,6 +37,7 @@ def flagDriftFromNorm(
norm_frac: float = 0.5,
metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
linkage_method: LinkageString = "single",
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -76,7 +79,8 @@ def flagDriftFromNorm(
The keyword gets passed on to scipy.hierarchy.linkage. See its documentation to learn more about the different
keywords (References [1]).
See wikipedia for an introduction to hierarchical clustering (References [2]).
- kwargs
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -122,7 +126,6 @@ def flagDriftFromNorm(
Introduction to Hierarchical clustering:
[2] https://en.wikipedia.org/wiki/Hierarchical_clustering
"""
-
data_to_flag = data[fields].to_df()
data_to_flag.dropna(inplace=True)
@@ -135,7 +138,7 @@ def flagDriftFromNorm(
drifters = detectDeviants(segment[1], metric, norm_spread, norm_frac, linkage_method, 'variables')
for var in drifters:
- flagger[segment[1].index, fields[var]] = kwargs['flag']
+ flagger[segment[1].index, fields[var]] = flag
return data, flagger
@@ -149,6 +152,7 @@ def flagDriftFromReference(
segment_freq: FreqString,
thresh: float,
metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -175,7 +179,8 @@ def flagDriftFromReference(
A distance function. It should be a function of 2 1-dimensional arrays and return a float scalar value.
This value is interpreted as the distance of the two input arrays. The default is the averaged manhatten metric.
See the Notes section to get an idea of why this could be a good choice.
- kwargs
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -211,7 +216,7 @@ def flagDriftFromReference(
dist = metric(segment[1].iloc[:, i].values, segment[1].loc[:, field].values)
if dist > thresh:
- flagger[segment[1].index, fields[i]] = kwargs['flag']
+ flagger[segment[1].index, fields[i]] = flag
return data, flagger
@@ -228,6 +233,7 @@ def flagDriftFromScaledNorm(
norm_frac: float = 0.5,
metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
linkage_method: LinkageString = "single",
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -277,7 +283,8 @@ def flagDriftFromScaledNorm(
The keyword gets passed on to scipy.hierarchy.linkage. See its documentation to learn more about the different
keywords (References [1]).
See wikipedia for an introduction to hierarchical clustering (References [2]).
- kwargs
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -327,7 +334,7 @@ def flagDriftFromScaledNorm(
drifters = detectDeviants(segment[1], metric, norm_spread, norm_frac, linkage_method, 'variables')
for var in drifters:
- flagger[segment[1].index, fields[var]] = kwargs['flag']
+ flagger[segment[1].index, fields[var]] = flag
return data, flagger
@@ -340,6 +347,7 @@ def correctExponentialDrift(
maint_data_field: ColumnName,
cal_mean: int = 5,
flag_maint_period: bool = False,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -390,6 +398,8 @@ def correctExponentialDrift(
directly before maintenance event. This values are needed for shift calibration. (see above description)
flag_maint_period : bool, default False
Whether or not to flag the values obtained while maintenance.
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -436,7 +446,7 @@ def correctExponentialDrift(
to_flag = drift_frame["drift_group"]
to_flag = to_flag.drop(to_flag[: maint_data.index[0]].index)
to_flag = to_flag.dropna()
- flagger[to_flag, field] = kwargs['flag']
+ flagger[to_flag, field] = flag
return data, flagger
@@ -487,7 +497,6 @@ def correctRegimeAnomaly(
x_date : bool, default False
If True, use "seconds from epoch" as x input to the model func, instead of "seconds from regime start".
-
Returns
-------
data : dios.DictOfSeries
@@ -592,7 +601,6 @@ def correctOffset(
start and right before the end of any regime is ignored when calculating a regimes mean for data correcture.
This is to account for the unrelyability of data near the changepoints of regimes.
-
Returns
-------
data : dios.DictOfSeries
@@ -600,7 +608,6 @@ def correctOffset(
Data values may have changed relatively to the data input.
flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
-
"""
data, flagger = copy(data, field, flagger, field + '_CPcluster')
data, flagger = assignChangePointCluster(
@@ -659,6 +666,7 @@ def flagRegimeAnomaly(
linkage_method: LinkageString = "single",
metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: np.abs(np.nanmean(x) - np.nanmean(y)),
norm_frac: float = 0.5,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -695,6 +703,8 @@ def flagRegimeAnomaly(
norm_frac : float
Has to be in [0,1]. Determines the minimum percentage of samples,
the "normal" group has to comprise to be the normal group actually.
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -714,6 +724,7 @@ def flagRegimeAnomaly(
norm_frac=norm_frac,
set_cluster=False,
set_flags=True,
+ flag=flag,
**kwargs
)
@@ -730,6 +741,7 @@ def assignRegimeAnomaly(
norm_frac: float = 0.5,
set_cluster: bool = True,
set_flags: bool = False,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -775,10 +787,11 @@ def assignRegimeAnomaly(
set_flags : bool, default True
Wheather or not to flag abnormal values (do not flag them, if you want to correct them
afterwards, becasue flagged values usually are not visible in further tests.).
+ flag : float, default BAD
+ flag to set.
Returns
-------
-
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
flagger : saqc.flagger.Flagger
@@ -792,7 +805,7 @@ def assignRegimeAnomaly(
if set_flags:
for p in plateaus:
- flagger[cluster_dios.iloc[:, p].index, field] = kwargs['flags']
+ flagger[cluster_dios.iloc[:, p].index, field] = flag
if set_cluster:
for p in plateaus:
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index 5c2b341a9..56c6a689c 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -76,6 +76,7 @@ def clearFlags(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs
flagUnflagged : set flag value at all unflagged positions
"""
if 'flag' in kwargs:
+ kwargs = {**kwargs} # copy
flag = kwargs.pop('flag')
warnings.warn(f'`flag={flag}` is ignored here.')
@@ -98,7 +99,7 @@ def flagUnflagged(
flagger : saqc.flagger.Flagger
A flagger object, holding flags and additional informations related to `data`.
flag : float, default BAD
- flag value to set, has NO default
+ flag value to set
kwargs : Dict
unused
@@ -149,7 +150,8 @@ def flagManual(
data: DictOfSeries, field: ColumnName, flagger: Flagger,
mdata: Union[pd.Series, pd.DataFrame, DictOfSeries],
mflag: Any = 1,
- method=Literal["plain", "ontime", "left-open", "right-open"],
+ method: Literal["plain", "ontime", "left-open", "right-open"] = 'plain',
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -172,6 +174,7 @@ def flagManual(
The "manually generated" data
mflag : scalar
The flag that indicates data points in `mdata`, of wich the projection in data should be flagged.
+
method : {'plain', 'ontime', 'left-open', 'right-open'}, default plain
Defines how mdata is projected on data. Except for the 'plain' method, the methods assume mdata to have an
index.
@@ -183,6 +186,9 @@ def flagManual(
the value at t_1 gets projected onto all data timestamps t with t_1 <= t < t_2.
* 'left-open': like 'right-open', but the projected interval now covers all t with t_1 < t <= t_2.
+ flag : float, default BAD
+ flag to set.
+
Returns
-------
data : original data
@@ -277,7 +283,7 @@ def flagManual(
mask = mdata == mflag
mask = mask.reindex(dat.index).fillna(False)
- flagger[mask, field] = kwargs['flag']
+ flagger[mask, field] = flag
return data, flagger
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index 1058da740..b8677f199 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -81,8 +81,14 @@ def _execGeneric(flagger: Flagger, data: DictOfSeries, func: Callable[[pd.Series
@register(masking='all', module="generic")
-def process(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd.Series], pd.Series],
- nodata: float = np.nan, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+def process(
+ data: DictOfSeries,
+ field: str,
+ flagger: Flagger,
+ func: Callable[[pd.Series], pd.Series],
+ nodata: float = np.nan,
+ **kwargs
+) -> Tuple[DictOfSeries, Flagger]:
"""
generate/process data with generically defined functions.
@@ -131,7 +137,6 @@ def process(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd
You also can pass numpy and pandas functions:
>>> lambda temperature, uncertainty: np.round(temperature) * np.sqrt(uncertainty)
-
"""
data[field] = _execGeneric(flagger, data, func, field, nodata).squeeze()
@@ -145,8 +150,15 @@ def process(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd
@register(masking='all', module="generic")
-def flag(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd.Series], pd.Series],
- nodata: float = np.nan, flag=BAD, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+def flag(
+ data: DictOfSeries,
+ field: str,
+ flagger: Flagger,
+ func: Callable[[pd.Series], pd.Series],
+ nodata: float = np.nan,
+ flag: float = BAD,
+ **kwargs
+) -> Tuple[DictOfSeries, Flagger]:
# TODO : fix docstring, check if all still works
"""
a function to flag a data column by evaluation of a generic expression.
@@ -181,6 +193,8 @@ def flag(data: DictOfSeries, field: str, flagger: Flagger, func: Callable[[pd.Se
See the examples section to learn more.
nodata : any, default np.nan
The value that indicates missing/invalid data
+ flag : float, default BAD
+ flag to set.
Returns
-------
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index 189995cc5..d8b2fcd26 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -36,6 +36,7 @@ def flagByStray(
partition_min: int = 11,
iter_start: float = 0.5,
alpha: float = 0.05,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -75,6 +76,9 @@ def flagByStray(
Level of significance by which it is tested, if a score might be drawn from another distribution, than the
majority of the data.
+ flag : float, default BAD
+ flag to set.
+
References
----------
[1] Talagala, P. D., Hyndman, R. J., & Smith-Miles, K. (2019). Anomaly detection in high dimensional data.
@@ -121,7 +125,7 @@ def flagByStray(
for iter_index in range(i_start - 1, sample_size):
if gaps[iter_index] > log_alpha * ghat[iter_index]:
index = partition.index[sorted_i[iter_index:]]
- flagger[index, field] = kwargs['flag']
+ flagger[index, field] = flag
break
return data, flagger
@@ -137,6 +141,7 @@ def _evalStrayLabels(
reduction_thresh: float = 3.5,
reduction_min_periods: int = 1,
at_least_one: bool = True,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -173,6 +178,8 @@ def _evalStrayLabels(
at_least_one : bool, default True
If none of the variables, the outlier label shall be reduced to, is an outlier with regard
to the test, all (True) or none (False) of the variables are flagged
+ flag : float, default BAD
+ flag to set.
References
----------
@@ -185,7 +192,7 @@ def _evalStrayLabels(
if reduction_range is None:
for field in to_flag_frame.columns:
- flagger[to_flag_frame.index, field] = kwargs['flag']
+ flagger[to_flag_frame.index, field] = flag
return data, flagger
for var in fields:
@@ -233,7 +240,7 @@ def _evalStrayLabels(
for field in to_flag_frame.columns:
col = to_flag_frame[field]
- flagger[col[col].index, field] = kwargs['flag']
+ flagger[col[col].index, field] = flag
return data, flagger
@@ -367,6 +374,7 @@ def flagMVScores(
reduction_drop_flagged: bool = False, # TODO: still a case ?
reduction_thresh: float = 3.5,
reduction_min_periods: int = 1,
+ flag: float = BAD,
**kwargs,
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -430,6 +438,8 @@ def flagMVScores(
Only effective when `reduction_range` is not ``None``.
Minimum number of meassurements necessarily present in a reduction interval for reduction actually to be
performed.
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -488,7 +498,9 @@ def flagMVScores(
partition_freq=stray_partition,
partition_min=stray_partition_min,
iter_start=iter_start,
- alpha=alpha, **kwargs)
+ alpha=alpha,
+ flag=flag,
+ **kwargs)
data, flagger = _evalStrayLabels(
data, 'kNN_scores', flagger,
@@ -496,7 +508,9 @@ def flagMVScores(
reduction_range=reduction_range,
reduction_drop_flagged=reduction_drop_flagged,
reduction_thresh=reduction_thresh,
- reduction_min_periods=reduction_min_periods, **kwargs)
+ reduction_min_periods=reduction_min_periods,
+ flag=flag,
+ **kwargs)
return data, flagger
@@ -514,6 +528,7 @@ def flagRaise(
min_slope: Optional[float] = None,
min_slope_weight: float = 0.8,
numba_boost: bool = True, # TODO: rm, not a user decision
+ flag: float = BAD,
**kwargs,
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -553,6 +568,9 @@ def flagRaise(
min_slope_weight : float, default 0.8
See third condition listed in the notes below.
numba_boost : bool, default True
+ deprecated ?
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -662,14 +680,20 @@ def flagRaise(
# check means against critical raise value:
to_flag = dataseries >= weighted_rolling_mean + (raise_series / mean_raise_factor)
to_flag &= raise_series.notna()
- flagger[to_flag[to_flag].index, field] = kwargs['flag']
+ flagger[to_flag[to_flag].index, field] = flag
return data, flagger
@register(masking='field', module="outliers")
def flagMAD(
- data: DictOfSeries, field: ColumnName, flagger: Flagger, window: FreqString, z: float = 3.5, **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flagger: Flagger,
+ window: FreqString,
+ z: float = 3.5,
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
The function represents an implementation of the modyfied Z-score outlier detection method.
@@ -690,6 +714,8 @@ def flagMAD(
Offset string. Denoting the windows size that the "Z-scored" values have to lie in.
z: float, default 3.5
The value the Z-score is tested against. Defaulting to 3.5 (Recommendation of [1])
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -721,7 +747,7 @@ def flagMAD(
index = mask.index
mask.loc[index < index[0] + pd.to_timedelta(window)] = False
- flagger[mask, field] = kwargs['flag']
+ flagger[mask, field] = flag
return data, flagger
@@ -735,6 +761,7 @@ def flagOffset(
window: Union[IntegerWindow, FreqString],
rel_thresh: Optional[float] = None,
numba_kickin: int = 200000, # TODO: rm, not a user decision
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -776,7 +803,8 @@ def flagOffset(
When there are detected more than `numba_kickin` incidents of potential spikes,
the pandas.rolling - part of computation gets "jitted" with numba.
Default value hast proven to be around the break even point between "jit-boost" and "jit-costs".
-
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -877,7 +905,7 @@ def flagOffset(
cresult = calcResult(result)
cresult = cresult[cresult].index
- flagger[cresult, field] = kwargs['flag']
+ flagger[cresult, field] = flag
return data, flagger
@@ -890,6 +918,7 @@ def flagByGrubbs(
alpha: float = 0.05,
min_periods: int = 8,
check_lagged: bool = False,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -927,6 +956,8 @@ def flagByGrubbs(
If True, every value gets checked twice for being an outlier. Ones in the initial rolling window and one more
time in a rolling window that is lagged by half the windows delimeter (winsz/2). Recommended for avoiding false
positives at the window edges. Only available when rolling with integer defined window size.
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -983,7 +1014,7 @@ def flagByGrubbs(
to_flag &= to_flag_lagged
- flagger[to_flag, field] = kwargs['flag']
+ flagger[to_flag, field] = flag
return data, flagger
@@ -994,6 +1025,7 @@ def flagRange(
flagger: Flagger,
min: float = -np.inf,
max: float = np.inf,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -1011,6 +1043,8 @@ def flagRange(
Lower bound for valid data.
max : float
Upper bound for valid data.
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -1024,7 +1058,7 @@ def flagRange(
# using .values is much faster
datacol = data[field].values
mask = (datacol < min) | (datacol > max)
- flagger[mask, field] = kwargs['flag']
+ flagger[mask, field] = flag
return data, flagger
@@ -1036,6 +1070,7 @@ def flagCrossStatistic(
fields: Sequence[ColumnName],
thresh: float,
cross_stat: Literal["modZscore", "Zscore"] = "modZscore",
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -1071,6 +1106,9 @@ def flagCrossStatistic(
* ``'Zscore'``: Score values by how many times the standard deviation they differ from the median.
See References [1]
+ flag : float, default BAD
+ flag to set.
+
Returns
-------
data : dios.DictOfSeries
@@ -1109,6 +1147,6 @@ def flagCrossStatistic(
mask = diff_scores > thresh
for var in fields:
- flagger[mask[var], var] = kwargs['flag']
+ flagger[mask[var], var] = flag
return data, flagger
diff --git a/saqc/funcs/pattern.py b/saqc/funcs/pattern.py
index a33cdceae..6562fd0d3 100644
--- a/saqc/funcs/pattern.py
+++ b/saqc/funcs/pattern.py
@@ -8,6 +8,7 @@ import pywt
from mlxtend.evaluate import permutation_test
from dios.dios import DictOfSeries
+from saqc.constants import *
from saqc.core.register import register
from saqc.flagger import Flagger
from saqc.lib.tools import customRoller
@@ -19,8 +20,9 @@ def flagPatternByDTW(
field: str,
flagger: Flagger,
ref_field: str,
- widths: Sequence[int]=(1, 2, 4, 8),
- waveform: str="mexh",
+ widths: Sequence[int] = (1, 2, 4, 8),
+ waveform: str = "mexh",
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -46,6 +48,8 @@ def flagPatternByDTW(
Widths for wavelet decomposition. [1] recommends a dyadic scale. Default: (1,2,4,8)
waveform: str.
Wavelet to be used for decomposition. Default: 'mexh'. See [2] for a list.
+ flag : float, default BAD
+ flag to set.
kwargs
@@ -94,7 +98,7 @@ def flagPatternByDTW(
sz = len(ref)
mask = customRoller(dat, window=sz, min_periods=sz).apply(isPattern, raw=True)
- flagger[mask, field] = kwargs['flag']
+ flagger[mask, field] = flag
return data, flagger
@@ -104,8 +108,9 @@ def flagPatternByWavelet(
field: str,
flagger: Flagger,
ref_field: str,
- max_distance: float=0.03,
- normalize: bool=True,
+ max_distance: float = 0.03,
+ normalize: bool = True,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
""" Pattern Recognition via Dynamic Time Warping.
@@ -130,9 +135,8 @@ def flagPatternByWavelet(
Maximum dtw-distance between partition and pattern, so that partition is recognized as pattern. Default: 0.03
normalize: boolean.
Normalizing dtw-distance (see [1]). Default: True
-
-
- kwargs
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -166,5 +170,5 @@ def flagPatternByWavelet(
sz = len(ref)
mask = customRoller(dat, window=sz, min_periods=sz).apply(isPattern, raw=True)
- flagger[mask, field] = kwargs['flag']
+ flagger[mask, field] = flag
return data, flagger
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index b5d2a109f..f69b12bb8 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -43,6 +43,7 @@ def aggregate(
value_func,
flag_func: Callable[[pd.Series], float] = np.nanmax,
method: Literal["fagg", "bagg", "nagg"] = "nagg",
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -94,6 +95,10 @@ def aggregate(
Specifies which intervals to be aggregated for a certain timestamp. (preceeding, succeeding or
"surrounding" interval). See description above for more details.
+ flag : float, default BAD
+ flag to set.
+
+
Returns
-------
data : dios.DictOfSeries
@@ -106,7 +111,13 @@ def aggregate(
data, flagger = copy(data, field, flagger, field + '_original')
return resample(
- data, field, flagger, freq=freq, agg_func=value_func, flag_agg_func=flag_func, method=method, **kwargs
+ data, field, flagger,
+ freq=freq,
+ agg_func=value_func,
+ flag_agg_func=flag_func,
+ method=method,
+ flag=flag,
+ **kwargs
)
@@ -674,7 +685,7 @@ def reindexFlags(
target_datcol = data[field]
target_flagscol = flagger[field]
- dummy = pd.Series(np.nan, target_flagscol.index)
+ dummy = pd.Series(np.nan, target_flagscol.index, dtype=float)
if method[-13:] == "interpolation":
ignore = _getChunkBounds(target_datcol, flagscol, freq)
diff --git a/saqc/funcs/residues.py b/saqc/funcs/residues.py
index 6abcfd2d6..0b0046bea 100644
--- a/saqc/funcs/residues.py
+++ b/saqc/funcs/residues.py
@@ -8,6 +8,7 @@ import numpy as np
from dios import DictOfSeries
+from saqc.constants import *
from saqc.core.register import register
from saqc.flagger import Flagger
from saqc.funcs.rolling import roll
@@ -24,6 +25,7 @@ def calculatePolynomialResidues(
numba: Literal[True, False, "auto"] = "auto", # TODO: rm, not a a user decision
eval_flags: bool = True, # TODO, not valid anymore, if still needed, maybe assign user-passed ``flag``?
min_periods: Optional[int] = 0,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
"""
@@ -88,6 +90,8 @@ def calculatePolynomialResidues(
fit to be performed. If there are not enough values, np.nan gets assigned. Default (0) results in fitting
regardless of the number of values present (results in overfitting for too sparse intervals). To automatically
set the minimum number of periods to the number of values in an offset defined window size, pass np.nan.
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -107,6 +111,7 @@ def calculatePolynomialResidues(
eval_flags=eval_flags,
min_periods=min_periods,
return_residues=True,
+ flag=flag,
**kwargs
)
@@ -121,6 +126,7 @@ def calculateRollingResidues(
eval_flags: bool = True,
min_periods: Optional[int] = 0,
center: bool = True,
+ flag: float = BAD,
**kwargs
) -> Tuple[DictOfSeries, Flagger]:
""" TODO: docstring needed"""
@@ -132,5 +138,6 @@ def calculateRollingResidues(
min_periods=min_periods,
center=center,
return_residues=True,
+ flag=flag,
**kwargs
)
diff --git a/saqc/funcs/rolling.py b/saqc/funcs/rolling.py
index 6a40c93c2..6d58dfbc6 100644
--- a/saqc/funcs/rolling.py
+++ b/saqc/funcs/rolling.py
@@ -5,9 +5,9 @@ from typing import Union, Callable
import numpy as np
import pandas as pd
-
from dios import DictOfSeries
+from saqc.constants import *
from saqc.core.register import register
from saqc.flagger import Flagger
from saqc.lib.tools import getFreqDelta
@@ -24,6 +24,7 @@ def roll(
min_periods: int=0,
center: bool=True,
return_residues=False, # TODO: this should not be public, a wrapper would be better
+ flag: float = BAD,
**kwargs
):
"""
@@ -59,6 +60,8 @@ def roll(
center : bool, default True
Wheather or not to center the window the mean is calculated of around the reference value. If False,
the reference value is placed to the right of the window (classic rolling mean with lag.)
+ flag : float, default BAD
+ flag to set.
Returns
-------
@@ -69,7 +72,6 @@ def roll(
The flagger object, holding flags and additional Informations related to `data`.
Flags values may have changed relatively to the flagger input.
"""
-
data = data.copy()
to_fit = data[field]
if to_fit.empty:
@@ -122,9 +124,6 @@ def roll(
data[field] = means
if eval_flags:
- # with the new flagger we dont have to care
- # about to set NaNs to the original flags anymore
-
# TODO: we does not get any flags here, because of masking=field
worst = flagger[field].rolling(winsz, center=True, min_periods=min_periods).max()
flagger[field] = worst
--
GitLab
From 233663c88adbb4a294e8b3cd6081fe9241532873 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 24 Mar 2021 00:10:54 +0100
Subject: [PATCH 081/180] mod gitlabci
---
.gitlab-ci.yml | 73 ++++++++++++++++++++++++++++++++------------------
1 file changed, 47 insertions(+), 26 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index eb0616e66..73f3f7e26 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,54 +1,49 @@
+# ===========================================================
+# preparation
+# ===========================================================
+
variables:
GIT_SUBMODULE_STRATEGY: recursive
-
default:
image: python:3.8
+ before_script:
+ - pip install --upgrade pip
+ - pip install pytest
+ - pip install -r requirements.txt
-before_script:
- - pip install --upgrade pip
- - pip install pytest
- - pip install -r requirements.txt
-
+# ===========================================================
+# normal jobs (non scheduled)
+# ===========================================================
# test saqc with python 3.7
python37:
stage: test
+ except:
+ - schedules
image: python:3.7
script:
- pytest tests/core tests/flagger tests/funcs
- - python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv --outfile /tmp/test.csv
+# - python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv --outfile /tmp/test.csv
# test saqc with python 3.8
python38:
stage: test
+ except:
+ - schedules
script:
- pytest tests/core tests/flagger tests/funcs
- - python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv --outfile /tmp/test.csv
-
-
-# test lib saqc
-testLib:
- stage: test
- script:
- - pytest tests/lib
-
-
-# fuzzy testing saqc
-fuzzy:
- allow_failure: true
- stage: test
- script:
- - pytest tests/fuzzy
+# - python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv --outfile /tmp/test.csv
# make (visual) coverage in gitlab merge request diff's
coverage:
- allow_failure: true
stage: test
-
+ except:
+ - schedules
+ allow_failure: true
script:
- pip install pytest-cov coverage
- pytest --cov=saqc tests/core tests/flagger tests/funcs
@@ -67,6 +62,10 @@ coverage:
# make html docu with sphinx
pages:
stage: deploy
+ only:
+ - develop
+ except:
+ - schedules
script:
- cd sphinx-doc/
- pip install -r requirements_sphinx.txt
@@ -75,5 +74,27 @@ pages:
artifacts:
paths:
- public
+
+
+# ===========================================================
+# scheduled jobs
+# ===========================================================
+
+# fuzzy testing saqc
+fuzzy:
+ stage: test
only:
- - develop
+ - schedules
+ allow_failure: true
+ script:
+ - pytest tests/fuzzy
+
+
+# test lib saqc
+testLib:
+ stage: test
+ only:
+ - schedules
+ script:
+ - pytest tests/lib
+
--
GitLab
From c0335442e3bfcd58f5db21a77c3a4c9d4289713b Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 24 Mar 2021 14:13:47 +0100
Subject: [PATCH 082/180] rm signatureMaker.py
---
signatureMaker.py | 207 ----------------------------------------------
1 file changed, 207 deletions(-)
delete mode 100644 signatureMaker.py
diff --git a/signatureMaker.py b/signatureMaker.py
deleted file mode 100644
index f31e9303f..000000000
--- a/signatureMaker.py
+++ /dev/null
@@ -1,207 +0,0 @@
-import re
-from typing import List
-
-from saqc.funcs import *
-from saqc.core.register import FUNC_MAP
-import os
-
-
-def start_with_exactly_N_spaces(line: str, N: int):
- return line.startswith(' ' * N) and not line.startswith(' ' * (N + 1))
-
-
-def find_original_signature(fh, func_name):
- """
- Extract the signature code from a file.
-
- Parameters
- ----------
- fh : file
- file descriptor, to read the code from
-
- func_name : str
- function name, of the signature in question
-
- Returns
- -------
- lines: list or None
- list of lines of code if found, otherwise None.
-
- """
- sig = []
- start = end = False
- for line in fh.readlines():
-
- # find start of signature
- if not start:
-
- if line.startswith(f'def {func_name}'):
- sig.append(line)
- start = True
- continue
-
- # find end of signature
- if '"""' in line or start_with_exactly_N_spaces(line, 4):
- end = True
- break # do not append line
-
- # found last line of signature
- if '->' in line:
- end = True
-
- sig.append(line)
-
- if end:
- break
-
- # if end or/and start was not found,
- # something went wrong
- if end is False:
- sig = None
-
- return sig
-
-
-def replace_core_signatures(core_code: List[str], sig_code: List[str], func_name: str, target_file):
- """
- Replace a signature in the core code with a signature from a module.
-
- Parameters
- ----------
- core_code : list
- lines of code, one by one
-
- sig_code : list
- lines of code, one by one (only the signature in question)
-
- func_name : str
- function name in question
-
- target_file : file
- file descriptor to write the modified code to
- """
- start = end = False
- for line in core_code:
-
- # append the rest of the file, the loop ends here
- if end is True:
- target_file.write(line)
- continue
-
- # find start of signature, loop starts here
- if not start:
-
- if line.startswith(f' def {func_name}'):
- start = True
-
- # insert the replacement
- for rline in sig_code:
- target_file.write(' ')
- target_file.write(rline)
-
- # start of sig, not found yet
- else:
- target_file.write(line)
- continue
-
- # found line after end of signature
- if '"""' in line or start_with_exactly_N_spaces(line, 8):
- end = True
- target_file.write(line)
- continue
-
- # found last line of signature
- if '->' in line:
- end = True
- continue
-
-
-def replace_datafieldflagger(lines):
- """
- Remove 'data' and 'flagger' from signature, and insert 'self' instead.
- """
- empty = re.compile(' *\n')
- data = re.compile('.*(data[=: ][^,]*, ?)') # eg. 'data: DictOfSeries,'
- flagger = re.compile('.*(flagger[=: ][^,]*, ?)') # eg. 'flagger: Flagger,'
- pattern_list = [data, flagger]
- i = 0
- replaced = []
-
- for line in lines:
-
- if i < len(pattern_list):
-
- # search for one patter after the other in the current line,
- # if any is NOT found, we stop and continued from there in the
- # next line (next loop outer integration)
- for j in range(i, len(pattern_list)):
- found = pattern_list[i].match(line)
- if found:
- # we replace the first match ('data') with 'self'
- line = line.replace(found[1], 'self, ' if i == 0 else '', 1)
- i += 1 # next pattern please
- else:
- break
-
- empty_line = empty.match(line)
- if empty_line:
- continue
-
- replaced.append(line)
-
- return replaced
-
-
-def autoreplace_signatures():
- """
- Replaces core-signatures with the module-signatures, one-by-one.
- """
- postfix = '_autosignature'
- saqc_path = 'saqc/core/modules/'
- touched_modules = []
-
- # one-by-one: we only process one signature at a time, this means
- # that we see most files multiple times.
- for name in FUNC_MAP:
- module, fname = name.split('.')
-
- with open(f'saqc/funcs/{module}.py', 'r') as fh:
- lines = find_original_signature(fh, fname)
-
- if lines is None:
- warnings.warn(f"end of signature of '{fname}' not found - ignoring")
- continue
-
- # modify original function signature
- lines = replace_datafieldflagger(lines)
- print(''.join(lines))
-
- # find the right file. If we already processed a signature
- # of the same module, we already have a modified file, so we
- # need to read and write the same.
- readfile = f'{saqc_path}{module}.py'
- writefile = f'{saqc_path}{module}{postfix}.py'
- if module in touched_modules:
- readfile = writefile
- else:
- touched_modules.append(module)
-
- # READ
- with open(readfile, 'r') as fh:
- readlines = fh.readlines()
-
- # WRITE
- # replace sig's in Saqc.module to a temporary file
- with open(writefile, 'w') as fh:
- replace_core_signatures(readlines, lines, fname, fh)
-
- # replace all original files with the temporary ones
- files = os.listdir('saqc/core/modules/')
- for new in files:
- if postfix in new:
- old = saqc_path + new.replace(postfix, '')
- new = saqc_path + new
- os.replace(new, old)
-
-if __name__ == '__main__':
- autoreplace_signatures()
--
GitLab
From 840a5c3d62ccd8db82c549c13938deeaf63e87ca Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 24 Mar 2021 15:43:21 +0100
Subject: [PATCH 083/180] fixed __main__.py, and .gitlab-ci.yml, added new
integration test
---
.gitlab-ci.yml | 5 ++--
saqc/__main__.py | 33 +++++++++++++--------------
saqc/flagger/flags.py | 1 +
tests/integration/__init__.py | 0
tests/integration/test_integration.py | 19 +++++++++++++++
5 files changed, 38 insertions(+), 20 deletions(-)
create mode 100644 tests/integration/__init__.py
create mode 100644 tests/integration/test_integration.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 73f3f7e26..490a4cf65 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -25,7 +25,7 @@ python37:
image: python:3.7
script:
- pytest tests/core tests/flagger tests/funcs
-# - python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv --outfile /tmp/test.csv
+ - python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv --outfile /tmp/test.csv
# test saqc with python 3.8
@@ -35,7 +35,7 @@ python38:
- schedules
script:
- pytest tests/core tests/flagger tests/funcs
-# - python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv --outfile /tmp/test.csv
+ - python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv --outfile /tmp/test.csv
# make (visual) coverage in gitlab merge request diff's
@@ -85,7 +85,6 @@ fuzzy:
stage: test
only:
- schedules
- allow_failure: true
script:
- pytest tests/fuzzy
diff --git a/saqc/__main__.py b/saqc/__main__.py
index 806377faa..b878c8237 100644
--- a/saqc/__main__.py
+++ b/saqc/__main__.py
@@ -2,6 +2,7 @@
# -*- coding: utf-8 -*-
import logging
+import warnings
from functools import partial
from pathlib import Path
@@ -11,18 +12,18 @@ import numpy as np
import pandas as pd
import pyarrow as pa
+from saqc.constants import *
from saqc.core import SaQC
-from saqc.flagger import CategoricalFlagger
-from saqc.flagger.dmpflagger import DmpFlagger
logger = logging.getLogger("SaQC")
-FLAGGERS = {
- "numeric": CategoricalFlagger([-1, 0, 1]),
- "category": CategoricalFlagger(["NIL", "OK", "BAD"]),
- "dmp": DmpFlagger(),
+SCHEMES = {
+ None: None,
+ "numeric": NotImplemented,
+ "category": NotImplemented,
+ "dmp": NotImplemented,
}
@@ -72,7 +73,7 @@ def writeData(writer_dict, df, fname):
)
@click.option("-o", "--outfile", type=click.Path(exists=False), help="path to the output file")
@click.option(
- "--flagger", default="category", type=click.Choice(FLAGGERS.keys()), help="the flagging scheme to use",
+ "--flagger", default=None, type=click.Choice(SCHEMES.keys()), help="the flagging scheme to use",
)
@click.option("--nodata", default=np.nan, help="nodata value")
@click.option(
@@ -81,27 +82,25 @@ def writeData(writer_dict, df, fname):
@click.option("--fail/--no-fail", default=True, help="whether to stop the program run on errors")
def main(config, data, flagger, outfile, nodata, log_level, fail):
+ if SCHEMES[flagger] is NotImplemented:
+ warnings.warn("flagger is currently not supported")
+
_setup_logging(log_level)
reader, writer = setupIO(nodata)
data = readData(reader, data)
- saqc = SaQC(flagger=FLAGGERS[flagger], data=data, nodata=nodata, error_policy="raise" if fail else "warn",)
+ saqc = SaQC(data=data, nodata=nodata, error_policy="raise" if fail else "warn",)
data_result, flagger_result = saqc.readConfig(config).getResult(raw=True)
if outfile:
data_result = data_result.to_df()
- flags = flagger_result.flags.to_df()
- flags_flagged = flagger_result.isFlagged().to_df()
-
- flags_out = flags.where((flags.isnull() | flags_flagged), flagger_result.GOOD)
- fields = {"data": data_result, "flags": flags_out}
+ flags = flagger_result.toFrame()
+ unflagged = (flags == UNFLAGGED) | flags.isna()
+ flags[unflagged] = GOOD
- if isinstance(flagger_result, DmpFlagger):
- fields["quality_flag"] = fields.pop("flags")
- fields["quality_comment"] = flagger_result.comments.to_df()
- fields["quality_cause"] = flagger_result.causes.to_df()
+ fields = {"data": data_result, "flags": flags}
out = (
pd.concat(fields.values(), axis=1, keys=fields.keys())
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index d40544a95..3b59e65ad 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -424,5 +424,6 @@ def appendHistory(flags: Flags, column, append_hist):
flags.history[column] = new_history
return flags
+
# for now we keep this name
Flagger = Flags
diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py
new file mode 100644
index 000000000..e88a90ab3
--- /dev/null
+++ b/tests/integration/test_integration.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+from click.testing import CliRunner
+import os
+
+
+def test__main__py():
+ import saqc.__main__
+
+ # if not run from project root
+ projpath = os.path.dirname(saqc.__file__) + '/../'
+
+ runner = CliRunner()
+ result = runner.invoke(
+ saqc.__main__.main, [
+ '--config', projpath + 'ressources/data/config_ci.csv',
+ '--data', projpath + 'ressources/data/data.csv',
+ '--outfile', '/tmp/test.csv', # the filesystem temp dir
+ ])
+ assert result.exit_code == 0, result.output
--
GitLab
From 2863bb55ab684977f433b8379626f08233f7db7f Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 24 Mar 2021 15:49:34 +0100
Subject: [PATCH 084/180] removed unused 'func_name'
---
saqc/core/core.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 3d4a7517c..9c3e4645c 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -241,7 +241,7 @@ class SaQC(FuncModules):
regex=regex,
)
- partial = func.bind(*fargs, **{"nodata": self._nodata, "func_name": func.name, **fkwargs})
+ partial = func.bind(*fargs, **{"nodata": self._nodata, **fkwargs})
out = self if inplace else self.copy(deep=True)
out._to_call.append((locator, control, partial))
@@ -309,7 +309,7 @@ def _warnForUnusedKwargs(func):
sig_kws = inspect.signature(func.func).parameters
# we need to ignore kws that are injected or by default hidden in ``**kwargs``
- ignore = ("nodata", "func_name", "flag", "to_mask")
+ ignore = ("nodata", "to_mask")
missing = []
for kw in func.keywords:
--
GitLab
From 09fa1f71aaccb11adc44ba6f8301d1bf0e9df668 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 24 Mar 2021 15:53:35 +0100
Subject: [PATCH 085/180] fixed deprecated todo in fuzzy testing
---
tests/fuzzy/test_functions.py | 5 -----
1 file changed, 5 deletions(-)
diff --git a/tests/fuzzy/test_functions.py b/tests/fuzzy/test_functions.py
index 09d1f8484..56fe671c8 100644
--- a/tests/fuzzy/test_functions.py
+++ b/tests/fuzzy/test_functions.py
@@ -14,11 +14,6 @@ from tests.fuzzy.init import MAX_EXAMPLES, functionKwargs
def callWontBreak(drawer, func_name: str):
func = FUNC_MAP[func_name]
kwargs = drawer.draw(functionKwargs(func))
-
- # TODO: workaround until `flag` is explicitly exposed in signature
- flag = drawer.draw(from_type(float))
- kwargs.setdefault('flag', flag)
-
func(**kwargs)
--
GitLab
From f9d011228e4061660d78172ea5228ca88ce8e2a0 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 24 Mar 2021 16:59:26 +0100
Subject: [PATCH 086/180] fixed History append
---
saqc/flagger/flags.py | 60 ++++++++++++++++------------------------
saqc/flagger/history.py | 42 ++++++++++++++++++++++++++++
saqc/funcs/resampling.py | 7 +++--
3 files changed, 70 insertions(+), 39 deletions(-)
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index 3b59e65ad..d2bd79def 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -356,18 +356,33 @@ def applyFunctionOnHistory(
Parameters
----------
- flags :
- column :
- hist_func :
- hist_kws :
- mask_func :
- mask_kws :
- last_column :
- func_handle_df :
+ flags : Flags
+ Flags object holding the History in question
+ column : str
+ name of the column holding the history in question
+ hist_func : callable
+ function to apply on `History.hist` (flags)
+ hist_kws : dict
+ hist-function keywords dict
+ mask_func : callable
+ function to apply on `History.mask` (force mask)
+ mask_kws : dict
+ mask-function keywords dict
+ last_column : pd.Series or None, default None
+ The last column to apply. If None, no extra column is appended.
+ func_handle_df : bool
+ If `True`, the whole History{.hist, .mask} are passed to the given functions, thus the
+ function must handle `pd.Dataframes` as first input. If `False`, each column is passed
+ separately, thus the functions must handle those.
+
+ Notes
+ -----
+ After the functions are called, all `NaN`'s in `History.mask` are replaced with `False`,
+ and the `.mask` is casted to bool, to ensure a consistent History.
Returns
-------
-
+ Copy of Flags with altered History (in column)
"""
flags = flags.copy()
history = flags.history[column]
@@ -398,32 +413,5 @@ def applyFunctionOnHistory(
return flags
-def appendHistory(flags: Flags, column, append_hist):
- """
- Function, specialized for used in deharm context.
-
-
- Parameters
- ----------
- flags
- field
- source
- merge_func
- merge_func_kws
- last_column
-
- Returns
- -------
-
- """
- flags = flags.copy()
- new_history = flags.history[column]
- for app_k in [k for k in append_hist.columns if k not in new_history.columns]:
- new_history.hist[app_k] = append_hist.hist[app_k]
- new_history.mask[app_k] = append_hist.mask[app_k]
- flags.history[column] = new_history
- return flags
-
-
# for now we keep this name
Flagger = Flags
diff --git a/saqc/flagger/history.py b/saqc/flagger/history.py
index 2acc8f22e..011a2dd41 100644
--- a/saqc/flagger/history.py
+++ b/saqc/flagger/history.py
@@ -393,3 +393,45 @@ class History:
raise ValueError('dtype must be float')
return obj
+
+
+def appendNewerHistory(original: History, newer: History) -> History:
+ """
+ Append all newer columns of a history to an other History.
+
+ The first N columns in the newer History are discarded, where N is the
+ number of columns in the original history.
+
+ The Histories must have same indexes, otherwise a `ValueError` is raised.
+
+ Parameters
+ ----------
+ original : History
+ The original History
+
+ newer : History
+ The newer History
+
+ Raises
+ ------
+ ValueError : if indexes of histories does not match.
+
+ Returns
+ -------
+ History with appended columns
+ """
+ original = original.copy()
+
+ if not original.index.equals(newer.index):
+ raise ValueError("Index of histories does not match")
+
+ n = len(original.columns)
+ append_hist = newer.hist.iloc[:, n:]
+ append_mask = newer.mask.iloc[:, n:]
+ original.hist.loc[:, append_hist.columns] = append_hist
+ original.mask.loc[:, append_mask.columns] = append_mask
+
+ assert original.columns.equals(pd.Index(range(len(original.columns))))
+ return original
+
+
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index f69b12bb8..33518f96a 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -13,12 +13,12 @@ from dios import DictOfSeries
from saqc.constants import *
from saqc.core.register import register, isflagged
-from saqc.flagger import Flagger, initFlagsLike, History
+from saqc.flagger.history import appendNewerHistory
+from saqc.flagger.flags import Flagger, applyFunctionOnHistory
from saqc.funcs.tools import copy, drop, rename
from saqc.funcs.interpolation import interpolateIndex, _SUPPORTED_METHODS
from saqc.lib.tools import evalFreqStr, getFreqDelta
from saqc.lib.ts_operators import shift2Freq, aggregate2Freq
-from saqc.flagger.flags import applyFunctionOnHistory, appendHistory
from saqc.lib.rolling import customRoller
logger = logging.getLogger("SaQC")
@@ -713,5 +713,6 @@ def reindexFlags(
raise ValueError(f"unknown method {method}")
tmp_flagger = applyFunctionOnHistory(flagger, source, func, func_kws, func, mask_kws, last_column=dummy)
- flagger = appendHistory(flagger, field, tmp_flagger.history[source])
+ new_hist = appendNewerHistory(flagger.history[field], tmp_flagger.history[source])
+ flagger.history[field] = new_hist
return data, flagger
--
GitLab
From 9c7b797c9d058d71294c40f1553d7a000d02df0e Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 24 Mar 2021 18:36:21 +0100
Subject: [PATCH 087/180] moved and simplified `applyFunctionOnHistory`
---
saqc/flagger/flags.py | 68 ---------------------------------
saqc/flagger/history.py | 76 +++++++++++++++++++++++++++++++++++--
saqc/funcs/interpolation.py | 6 +--
saqc/funcs/resampling.py | 13 +++----
4 files changed, 82 insertions(+), 81 deletions(-)
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index d2bd79def..11e5f598f 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -345,73 +345,5 @@ def initFlagsLike(
return Flags(result)
-def applyFunctionOnHistory(
- flags: Flags, column, hist_func, hist_kws, mask_func, mask_kws, last_column=None, func_handle_df=False
-):
- """
- Apply function on history.
-
- Two functions must be given. Both are called for each column in the History. One on History.hist, the
- other on History.mask. Both take a pd.Series as first arg, which is the column from hist or mask respectively.
-
- Parameters
- ----------
- flags : Flags
- Flags object holding the History in question
- column : str
- name of the column holding the history in question
- hist_func : callable
- function to apply on `History.hist` (flags)
- hist_kws : dict
- hist-function keywords dict
- mask_func : callable
- function to apply on `History.mask` (force mask)
- mask_kws : dict
- mask-function keywords dict
- last_column : pd.Series or None, default None
- The last column to apply. If None, no extra column is appended.
- func_handle_df : bool
- If `True`, the whole History{.hist, .mask} are passed to the given functions, thus the
- function must handle `pd.Dataframes` as first input. If `False`, each column is passed
- separately, thus the functions must handle those.
-
- Notes
- -----
- After the functions are called, all `NaN`'s in `History.mask` are replaced with `False`,
- and the `.mask` is casted to bool, to ensure a consistent History.
-
- Returns
- -------
- Copy of Flags with altered History (in column)
- """
- flags = flags.copy()
- history = flags.history[column]
- new_history = History()
-
- if func_handle_df:
- history.hist = hist_func(history.hist, **hist_kws)
- history.mask = hist_func(history.mask, **mask_kws)
-
- else:
- for pos in history.columns:
- new_history.hist[pos] = hist_func(history.hist[pos], **hist_kws)
- new_history.mask[pos] = mask_func(history.mask[pos], **mask_kws)
-
- # handle unstable state
- if last_column is None:
- new_history.mask.iloc[:, -1:] = True
- else:
- if isinstance(last_column, str) and last_column == 'dummy':
- last_column = pd.Series(UNTOUCHED, index=new_history.index, dtype=float)
-
- new_history.append(last_column, force=True)
-
- # assure a boolean mask
- new_history.mask = new_history.mask.fillna(False).astype(bool)
-
- flags.history[column] = new_history
- return flags
-
-
# for now we keep this name
Flagger = Flags
diff --git a/saqc/flagger/history.py b/saqc/flagger/history.py
index 011a2dd41..0f122db1f 100644
--- a/saqc/flagger/history.py
+++ b/saqc/flagger/history.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from __future__ import annotations
-from typing import Tuple, Type
+from typing import Tuple, Type, Union, Literal
import pandas as pd
import numpy as np
from saqc.constants import *
@@ -420,8 +420,6 @@ def appendNewerHistory(original: History, newer: History) -> History:
-------
History with appended columns
"""
- original = original.copy()
-
if not original.index.equals(newer.index):
raise ValueError("Index of histories does not match")
@@ -432,6 +430,78 @@ def appendNewerHistory(original: History, newer: History) -> History:
original.mask.loc[:, append_mask.columns] = append_mask
assert original.columns.equals(pd.Index(range(len(original.columns))))
+
return original
+def applyFunctionOnHistory(
+ history: History,
+ hist_func: callable,
+ hist_kws: dict,
+ mask_func: callable,
+ mask_kws: dict,
+ last_column: Union[pd.Series, Literal['dummy'], None] = None,
+ func_handle_df: bool = False,
+):
+ """
+ Apply function on each column in history.
+
+ Two functions must be given. Both are called for each column in the History unless ``func_handle_df=True`` is
+ given. One is called on ``History.hist``, the other on ``History.mask``.
+ Both function must take a pd.Series as first arg, which is the column from hist or mask respectively. If
+ ``func_handle_df=True`` each functions must take a ``pd.DataFrame`` as first argument, holding all columns
+ at once. The function must return same type as first argument.
+
+ Parameters
+ ----------
+ history : History
+ History object to alter
+ hist_func : callable
+ function to apply on `History.hist` (flags DataFrame)
+ hist_kws : dict
+ hist-function keywords dict
+ mask_func : callable
+ function to apply on `History.mask` (force mask DataFrame)
+ mask_kws : dict
+ mask-function keywords dict
+ last_column : pd.Series or None, default None
+ The last column to apply. If None, no extra column is appended.
+ func_handle_df : bool
+ If `True`, the whole History{.hist, .mask} are passed to the given functions, thus the
+ function must handle `pd.Dataframes` as first input. If `False`, each column is passed
+ separately, thus the functions must handle those.
+
+ Notes
+ -----
+ After the functions are called, all `NaN`'s in `History.mask` are replaced with `False`,
+ and the `.mask` is casted to bool, to ensure a consistent History.
+
+ Returns
+ -------
+ history with altered columns
+ """
+ new_history = History()
+
+ if func_handle_df:
+ history.hist = hist_func(history.hist, **hist_kws)
+ history.mask = hist_func(history.mask, **mask_kws)
+
+ else:
+ for pos in history.columns:
+ new_history.hist[pos] = hist_func(history.hist[pos], **hist_kws)
+ new_history.mask[pos] = mask_func(history.mask[pos], **mask_kws)
+
+ # handle unstable state
+ if last_column is None:
+ new_history.mask.iloc[:, -1:] = True
+ else:
+ if isinstance(last_column, str) and last_column == 'dummy':
+ last_column = pd.Series(UNTOUCHED, index=new_history.index, dtype=float)
+
+ new_history.append(last_column, force=True)
+
+ # assure a boolean mask
+ new_history.mask = new_history.mask.fillna(False).astype(bool)
+
+ return new_history
+
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index c5d4f0768..5c9e8974f 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -12,7 +12,7 @@ from dios import DictOfSeries
from saqc.constants import *
from saqc.core.register import register, isflagged
from saqc.flagger import Flagger
-from saqc.flagger.flags import applyFunctionOnHistory
+from saqc.flagger.history import applyFunctionOnHistory
from saqc.lib.ts_operators import interpolateNANs
_SUPPORTED_METHODS = Literal[
@@ -278,8 +278,8 @@ def interpolateIndex(
data[field] = inter_data[grid_index]
# do the reshaping on the history
- flagger = applyFunctionOnHistory(
- flagger, field,
+ flagger.history[field] = applyFunctionOnHistory(
+ flagger.history[field],
hist_func=_resampleOverlapping, hist_kws=dict(freq=freq, fill_value=UNFLAGGED),
mask_func=_resampleOverlapping, mask_kws=dict(freq=freq, fill_value=False),
last_column='dummy'
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 33518f96a..a459e22b7 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -13,8 +13,8 @@ from dios import DictOfSeries
from saqc.constants import *
from saqc.core.register import register, isflagged
-from saqc.flagger.history import appendNewerHistory
-from saqc.flagger.flags import Flagger, applyFunctionOnHistory
+from saqc.flagger.history import appendNewerHistory, applyFunctionOnHistory
+from saqc.flagger.flags import Flagger
from saqc.funcs.tools import copy, drop, rename
from saqc.funcs.interpolation import interpolateIndex, _SUPPORTED_METHODS
from saqc.lib.tools import evalFreqStr, getFreqDelta
@@ -540,8 +540,8 @@ def resample(
max_invalid_consec=max_invalid_consec_f,
)
- flagger = applyFunctionOnHistory(
- flagger, field,
+ flagger.history[field] = applyFunctionOnHistory(
+ flagger.history[field],
hist_func=aggregate2Freq, hist_kws=kws,
mask_func=aggregate2Freq, mask_kws=kws,
last_column='dummy'
@@ -712,7 +712,6 @@ def reindexFlags(
else:
raise ValueError(f"unknown method {method}")
- tmp_flagger = applyFunctionOnHistory(flagger, source, func, func_kws, func, mask_kws, last_column=dummy)
- new_hist = appendNewerHistory(flagger.history[field], tmp_flagger.history[source])
- flagger.history[field] = new_hist
+ history = applyFunctionOnHistory(flagger.history[source], func, func_kws, func, mask_kws, last_column=dummy)
+ flagger.history[field] = appendNewerHistory(flagger.history[field], history)
return data, flagger
--
GitLab
From 75b1e52f15e84df5f52b7c1f699e32c216e11fe2 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Thu, 25 Mar 2021 14:32:48 +0100
Subject: [PATCH 088/180] fixed initialize bug
---
saqc/core/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 9c3e4645c..6d5ecbf16 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -104,7 +104,7 @@ class SaQC(FuncModules):
def __init__(self, data, flags=None, nodata=np.nan, to_mask=None, error_policy="raise"):
super().__init__(self)
- data, flagger = _prepInput(data, flags)
+ data, flags = _prepInput(data, flags)
self._data = data
self._nodata = nodata
self._to_mask = to_mask
--
GitLab
From b432eef1483721c6e6dd3bf7fbe1f9f7d20fb193 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Thu, 25 Mar 2021 15:56:07 +0100
Subject: [PATCH 089/180] merged `appendNewerHistory` and `History.append`
---
saqc/flagger/history.py | 125 +++++++++++++++++++++++----------------
saqc/funcs/resampling.py | 4 +-
2 files changed, 75 insertions(+), 54 deletions(-)
diff --git a/saqc/flagger/history.py b/saqc/flagger/history.py
index 0f122db1f..e5220ddda 100644
--- a/saqc/flagger/history.py
+++ b/saqc/flagger/history.py
@@ -167,37 +167,97 @@ class History:
return self
- def append(self, value: pd.Series, force=False) -> History:
+ def append(self, value: Union[pd.Series, History], force=False) -> History:
"""
Create a new FH column and insert given pd.Series to it.
Parameters
----------
- value : pd.Series
- the data to append. Must have dtype float and the index must
- match the index of the FH.
+ value : pd.Series or History
+ The data to append. Must have dtype float and the index must
+ match the index of the History.
force : bool, default False
- if True the internal mask is updated in a way that the currently
- set value (series values) will be returned if ``History.max()``
- is called. This apply for all valid values (not ``np.Nan`` and
- not ``-np.inf``).
+
+ If ``value`` is a ``pd.Series``:
+
+ - ``force=True`` : the internal mask is updated in a way that the currently
+ set values gets the highest priority in the current history.
+ This means, these values are guaranteed to be returned if ``History.max()``
+ is called, until newer possibly higher flags are set. Bear in mind
+ that this never apply for `Nan`-values.
+ - ``force=False`` : values are not treated special.
+
+ If ``value`` is a ``History``:
+
+ - ``force=True`` : All columns are appended to the existing history.
+ - ``force=False`` : Only columns that are `newer` are appended. This means
+ the first ``N`` columns of the passed history are discarded, where ``N`` is the
+ number of existing columns in the current history.
+
+ Returns
+ -------
+ history with appended series
Raises
------
ValueError: on index miss-match or wrong dtype
TypeError: if value is not pd.Series
+ """
+ if isinstance(value, History):
+ return self._appendHistory(value, force=force)
+
+ value = self._validate_value(value)
+ if len(self) > 0 and not value.index.equals(self.index):
+ raise ValueError("Index must be equal to history index")
+
+ self._insert(value, pos=len(self), force=force)
+ return self
+
+ def _appendHistory(self, value: History, force: bool = False):
+ """
+ Append multiple columns of a history to self.
+
+ Parameters
+ ----------
+ value : History
+ Holding the columns to append
+ force : bool
+ If False, the first `N` columns in the passed History are discarded, where
+ `N` is the number of columns in the original history.
+ If ``force=True`` all columns are appended.
Returns
-------
- History: FH with appended series
+ History with appended columns.
+
+ Raises
+ ------
+ ValueError : If the index of the passed history does not match.
+
+ Notes
+ -----
+ This ignores the column names of the passed History.
"""
- s = self._validate_value(value)
+ self._validate_hist_with_mask(value.hist, value.mask)
+ if len(self) > 0 and not value.index.equals(self.index):
+ raise ValueError("Index must be equal to history index")
- if len(self) > 0 and not s.index.equals(self.index):
- raise ValueError("Index must be equal to FlagHistory index")
+ n = len(self.columns)
+ value_hist = value.hist
+ value_mask = value.mask
- self._insert(value, pos=len(self), force=force)
+ if not force:
+ value_hist = value_hist.iloc[:, n:]
+ value_mask = value_mask.iloc[:, n:]
+
+ # rename columns, to avoid ``pd.DataFrame.loc`` become confused
+ columns = pd.Index(range(n, len(value_hist.columns) + 1))
+ value_hist.columns = columns
+ value_mask.columns = columns
+
+ self.hist.loc[:, columns] = value_hist.copy()
+ self.mask.loc[:, columns] = value_mask.copy()
return self
def squeeze(self, n: int) -> History:
@@ -395,45 +455,6 @@ class History:
return obj
-def appendNewerHistory(original: History, newer: History) -> History:
- """
- Append all newer columns of a history to an other History.
-
- The first N columns in the newer History are discarded, where N is the
- number of columns in the original history.
-
- The Histories must have same indexes, otherwise a `ValueError` is raised.
-
- Parameters
- ----------
- original : History
- The original History
-
- newer : History
- The newer History
-
- Raises
- ------
- ValueError : if indexes of histories does not match.
-
- Returns
- -------
- History with appended columns
- """
- if not original.index.equals(newer.index):
- raise ValueError("Index of histories does not match")
-
- n = len(original.columns)
- append_hist = newer.hist.iloc[:, n:]
- append_mask = newer.mask.iloc[:, n:]
- original.hist.loc[:, append_hist.columns] = append_hist
- original.mask.loc[:, append_mask.columns] = append_mask
-
- assert original.columns.equals(pd.Index(range(len(original.columns))))
-
- return original
-
-
def applyFunctionOnHistory(
history: History,
hist_func: callable,
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index a459e22b7..e64ec56f8 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -13,7 +13,7 @@ from dios import DictOfSeries
from saqc.constants import *
from saqc.core.register import register, isflagged
-from saqc.flagger.history import appendNewerHistory, applyFunctionOnHistory
+from saqc.flagger.history import applyFunctionOnHistory
from saqc.flagger.flags import Flagger
from saqc.funcs.tools import copy, drop, rename
from saqc.funcs.interpolation import interpolateIndex, _SUPPORTED_METHODS
@@ -713,5 +713,5 @@ def reindexFlags(
raise ValueError(f"unknown method {method}")
history = applyFunctionOnHistory(flagger.history[source], func, func_kws, func, mask_kws, last_column=dummy)
- flagger.history[field] = appendNewerHistory(flagger.history[field], history)
+ flagger.history[field] = flagger.history[field].append(history, force=False)
return data, flagger
--
GitLab
From bfa930e35d253ec81f01b5a42fee3d14a658c03f Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Thu, 25 Mar 2021 16:14:23 +0100
Subject: [PATCH 090/180] killed some snakes, hiding in the forest of saqc
---
saqc/__main__.py | 4 ++--
saqc/flagger/flags.py | 2 +-
saqc/flagger/history.py | 16 ++++++++--------
saqc/funcs/drift.py | 4 ++--
4 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/saqc/__main__.py b/saqc/__main__.py
index b878c8237..8fbb3cb7c 100644
--- a/saqc/__main__.py
+++ b/saqc/__main__.py
@@ -27,7 +27,7 @@ SCHEMES = {
}
-def _setup_logging(loglvl):
+def _setupLogging(loglvl):
logger.setLevel(loglvl)
handler = logging.StreamHandler()
formatter = logging.Formatter("[%(asctime)s][%(name)s][%(levelname)s]: %(message)s")
@@ -85,7 +85,7 @@ def main(config, data, flagger, outfile, nodata, log_level, fail):
if SCHEMES[flagger] is NotImplemented:
warnings.warn("flagger is currently not supported")
- _setup_logging(log_level)
+ _setupLogging(log_level)
reader, writer = setupIO(nodata)
data = readData(reader, data)
diff --git a/saqc/flagger/flags.py b/saqc/flagger/flags.py
index 11e5f598f..63d3a505f 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/flagger/flags.py
@@ -40,7 +40,7 @@ class _HistAccess:
if not isinstance(value, History):
raise TypeError("Not a History")
- History._validate_hist_with_mask(value.hist, value.mask)
+ History._validateHistWithMask(value.hist, value.mask)
self.obj._data[key] = value
self.obj._cache.pop(key, None)
diff --git a/saqc/flagger/history.py b/saqc/flagger/history.py
index e5220ddda..8ac0552a7 100644
--- a/saqc/flagger/history.py
+++ b/saqc/flagger/history.py
@@ -65,11 +65,11 @@ class History:
raise ValueError("Cannot take 'mask' without 'hist'")
elif hist is not None and mask is None:
- hist = self._validate_hist(hist)
+ hist = self._validateHist(hist)
mask = pd.DataFrame(True, index=hist.index, columns=hist.columns)
else:
- hist, mask = self._validate_hist_with_mask(hist, mask)
+ hist, mask = self._validateHistWithMask(hist, mask)
if copy:
hist = hist.copy()
@@ -207,7 +207,7 @@ class History:
if isinstance(value, History):
return self._appendHistory(value, force=force)
- value = self._validate_value(value)
+ value = self._validateValue(value)
if len(self) > 0 and not value.index.equals(self.index):
raise ValueError("Index must be equal to history index")
@@ -239,7 +239,7 @@ class History:
-----
This ignores the column names of the passed History.
"""
- self._validate_hist_with_mask(value.hist, value.mask)
+ self._validateHistWithMask(value.hist, value.mask)
if len(self) > 0 and not value.index.equals(self.index):
raise ValueError("Index must be equal to history index")
@@ -394,13 +394,13 @@ class History:
#
@staticmethod
- def _validate_hist_with_mask(obj: pd.DataFrame, mask: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]:
+ def _validateHistWithMask(obj: pd.DataFrame, mask: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]:
"""
check type, columns, index, dtype and if the mask fits the obj.
"""
# check hist
- History._validate_hist(obj)
+ History._validateHist(obj)
# check mask
if not isinstance(mask, pd.DataFrame):
@@ -422,7 +422,7 @@ class History:
return obj, mask
@staticmethod
- def _validate_hist(obj: pd.DataFrame) -> pd.DataFrame:
+ def _validateHist(obj: pd.DataFrame) -> pd.DataFrame:
"""
check type, columns, dtype of obj.
"""
@@ -442,7 +442,7 @@ class History:
return obj
@staticmethod
- def _validate_value(obj: pd.Series) -> pd.Series:
+ def _validateValue(obj: pd.Series) -> pd.Series:
"""
index is not checked !
"""
diff --git a/saqc/funcs/drift.py b/saqc/funcs/drift.py
index d8605f67d..2dd2fe7c4 100644
--- a/saqc/funcs/drift.py
+++ b/saqc/funcs/drift.py
@@ -433,7 +433,7 @@ def correctExponentialDrift(
for k, group in drift_grouper:
dataSeries = group[to_correct.name]
- dataFit, dataShiftTarget = _drift_fit(dataSeries, shift_targets.loc[k, :][0], cal_mean)
+ dataFit, dataShiftTarget = _driftFit(dataSeries, shift_targets.loc[k, :][0], cal_mean)
dataFit = pd.Series(dataFit, index=group.index)
dataShiftTarget = pd.Series(dataShiftTarget, index=group.index)
dataShiftVektor = dataShiftTarget - dataFit
@@ -628,7 +628,7 @@ def correctOffset(
return data, flagger
-def _drift_fit(x, shift_target, cal_mean):
+def _driftFit(x, shift_target, cal_mean):
x_index = x.index - x.index[0]
x_data = x_index.total_seconds().values
x_data = x_data / x_data[-1]
--
GitLab
From 21d635b051aceed6f856a9f5c7956d9fd34d67e8 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Thu, 25 Mar 2021 21:16:09 +0100
Subject: [PATCH 091/180] make `isflagged` private
---
saqc/core/register.py | 4 ++--
saqc/funcs/interpolation.py | 4 ++--
saqc/funcs/resampling.py | 8 ++++----
3 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 39c1d4b14..8cd859530 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -238,7 +238,7 @@ def _maskData(data, flagger, columns, thresh) -> Tuple[dios.DictOfSeries, dios.D
# we use numpy here because it is faster
for c in columns:
- col_mask = isflagged(flagger[c].to_numpy(), thresh)
+ col_mask = _isflagged(flagger[c].to_numpy(), thresh)
if any(col_mask):
col_data = data[c].to_numpy(dtype=np.float64)
@@ -250,7 +250,7 @@ def _maskData(data, flagger, columns, thresh) -> Tuple[dios.DictOfSeries, dios.D
return data, mask
-def isflagged(flags: Union[np.array, pd.Series], thresh: float) -> Union[np.array, pd.Series]:
+def _isflagged(flags: Union[np.array, pd.Series], thresh: float) -> Union[np.array, pd.Series]:
"""
Return a mask of flags accordingly to `thresh`. Return type is same as flags.
"""
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index 5c9e8974f..a7880f4b0 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -10,7 +10,7 @@ import pandas as pd
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core.register import register, isflagged
+from saqc.core.register import register, _isflagged
from saqc.flagger import Flagger
from saqc.flagger.history import applyFunctionOnHistory
from saqc.lib.ts_operators import interpolateNANs
@@ -248,7 +248,7 @@ def interpolateIndex(
start, end = datcol.index[0].floor(freq), datcol.index[-1].ceil(freq)
grid_index = pd.date_range(start=start, end=end, freq=freq, name=datcol.index.name)
- flagged = isflagged(flagger[field], kwargs['to_mask'])
+ flagged = _isflagged(flagger[field], kwargs['to_mask'])
# drop all points that hold no relevant grid information
datcol = datcol[~flagged].dropna()
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index e64ec56f8..0a796d726 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -12,7 +12,7 @@ import pandas as pd
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core.register import register, isflagged
+from saqc.core.register import register, _isflagged
from saqc.flagger.history import applyFunctionOnHistory
from saqc.flagger.flags import Flagger
from saqc.funcs.tools import copy, drop, rename
@@ -393,7 +393,7 @@ def _shift(
--------
shift : Main caller, docstring
"""
- flagged = isflagged(flagger[field], kwargs['to_mask'])
+ flagged = _isflagged(flagger[field], kwargs['to_mask'])
datcol = data[field]
datcol[flagged] = np.nan
freq = evalFreqStr(freq, freq_check, datcol.index)
@@ -516,7 +516,7 @@ def resample(
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
- flagged = isflagged(flagger[field], kwargs['to_mask'])
+ flagged = _isflagged(flagger[field], kwargs['to_mask'])
datcol = data[field]
datcol[flagged] = np.nan
freq = evalFreqStr(freq, freq_check, datcol.index)
@@ -701,7 +701,7 @@ def reindexFlags(
mask_kws = func_kws
elif method[-5:] == "shift":
- drop_mask = (target_datcol.isna() | isflagged(target_flagscol, kwargs['to_mask']))
+ drop_mask = (target_datcol.isna() | _isflagged(target_flagscol, kwargs['to_mask']))
projection_method = METHOD2ARGS[method][0]
tolerance = METHOD2ARGS[method][1](freq)
func = _inverseShift
--
GitLab
From 85032515bc57ac22cb0a3631518e4d02931c11ff Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 26 Mar 2021 00:09:28 +0100
Subject: [PATCH 092/180] moved flags.py and history.py to core
---
saqc/__init__.py | 4 +---
saqc/core/__init__.py | 1 +
saqc/core/core.py | 2 +-
saqc/{flagger => core}/flags.py | 12 +++++-------
saqc/{flagger => core}/history.py | 0
saqc/core/modules/breaks.py | 2 +-
saqc/core/modules/changepoints.py | 2 +-
saqc/core/modules/constants.py | 2 +-
saqc/core/modules/curvefit.py | 2 +-
saqc/core/modules/flagtools.py | 2 +-
saqc/core/modules/generic.py | 3 ++-
saqc/core/modules/interpolation.py | 2 +-
saqc/core/modules/outliers.py | 2 +-
saqc/core/modules/pattern.py | 2 +-
saqc/core/modules/resampling.py | 2 +-
saqc/core/modules/residues.py | 2 +-
saqc/core/modules/scores.py | 2 +-
saqc/core/modules/tools.py | 2 +-
saqc/core/modules/transformation.py | 2 +-
saqc/core/reader.py | 6 ++----
saqc/core/register.py | 5 ++---
saqc/core/visitor.py | 1 -
saqc/flagger/__init__.py | 5 -----
saqc/funcs/breaks.py | 3 +--
saqc/funcs/changepoints.py | 3 +--
saqc/funcs/constants.py | 3 +--
saqc/funcs/curvefit.py | 3 +--
saqc/funcs/drift.py | 2 +-
saqc/funcs/flagtools.py | 5 +----
saqc/funcs/generic.py | 3 +--
saqc/funcs/interpolation.py | 11 ++++-------
saqc/funcs/outliers.py | 18 +++++-------------
saqc/funcs/pattern.py | 3 +--
saqc/funcs/resampling.py | 19 ++++++++-----------
saqc/funcs/residues.py | 5 +----
saqc/funcs/rolling.py | 4 +---
saqc/funcs/scores.py | 7 ++-----
saqc/funcs/tools.py | 5 +----
saqc/funcs/transformation.py | 5 +----
saqc/lib/plotting.py | 2 +-
saqc/lib/tools.py | 2 +-
saqc/lib/ts_operators.py | 3 ---
saqc/lib/types.py | 7 ++++---
tests/common.py | 2 +-
tests/core/test_core.py | 2 +-
tests/core/test_creation.py | 2 +-
tests/flagger/test_flags.py | 2 +-
tests/flagger/test_history.py | 2 +-
tests/funcs/test_constants_detection.py | 2 +-
tests/funcs/test_functions.py | 2 +-
tests/funcs/test_generic_config_functions.py | 2 +-
tests/funcs/test_harm_funcs.py | 2 +-
tests/funcs/test_modelling.py | 2 +-
tests/funcs/test_pattern_rec.py | 2 +-
tests/funcs/test_proc_functions.py | 2 +-
tests/funcs/test_spikes_detection.py | 2 +-
tests/fuzzy/init.py | 2 +-
tests/fuzzy/test_masking.py | 2 +-
58 files changed, 79 insertions(+), 129 deletions(-)
rename saqc/{flagger => core}/flags.py (99%)
rename saqc/{flagger => core}/history.py (100%)
delete mode 100644 saqc/flagger/__init__.py
diff --git a/saqc/__init__.py b/saqc/__init__.py
index d7155f47a..6262ae74d 100644
--- a/saqc/__init__.py
+++ b/saqc/__init__.py
@@ -5,6 +5,4 @@ __version__ = "1.4"
# import order: from small to big
from saqc.constants import *
-from saqc.flagger import *
-from saqc.core.register import register
-from saqc.core.core import SaQC
+from saqc.core import register, initFlagsLike, Flags, SaQC
diff --git a/saqc/core/__init__.py b/saqc/core/__init__.py
index 097236acb..0b4aacf33 100644
--- a/saqc/core/__init__.py
+++ b/saqc/core/__init__.py
@@ -2,4 +2,5 @@
# -*- coding: utf-8 -*-
from saqc.core.register import register
+from saqc.core.flags import Flags, initFlagsLike
from saqc.core.core import SaQC, logger
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 6d5ecbf16..2fe5c6e11 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -18,7 +18,7 @@ import timeit
import inspect
from saqc.constants import *
-from saqc.flagger import initFlagsLike, Flagger
+from saqc.core.flags import initFlagsLike, Flags as Flagger
from saqc.core.lib import APIController, ColumnSelector
from saqc.core.register import FUNC_MAP, SaQCFunction
from saqc.core.modules import FuncModules
diff --git a/saqc/flagger/flags.py b/saqc/core/flags.py
similarity index 99%
rename from saqc/flagger/flags.py
rename to saqc/core/flags.py
index 63d3a505f..1d95adf31 100644
--- a/saqc/flagger/flags.py
+++ b/saqc/core/flags.py
@@ -2,12 +2,14 @@
from __future__ import annotations
-import dios
-from saqc.constants import *
-from saqc.flagger.history import History
import pandas as pd
+import dios
from typing import Union, Dict, DefaultDict, Optional, Type, Tuple, Iterable
+from saqc.constants import *
+from saqc.core.history import History
+
+
_VAL = Union[pd.Series, History]
DictLike = Union[
pd.DataFrame,
@@ -343,7 +345,3 @@ def initFlagsLike(
result[k] = History(item)
return Flags(result)
-
-
-# for now we keep this name
-Flagger = Flags
diff --git a/saqc/flagger/history.py b/saqc/core/history.py
similarity index 100%
rename from saqc/flagger/history.py
rename to saqc/core/history.py
diff --git a/saqc/core/modules/breaks.py b/saqc/core/modules/breaks.py
index b07edba09..6fab21ff3 100644
--- a/saqc/core/modules/breaks.py
+++ b/saqc/core/modules/breaks.py
@@ -6,8 +6,8 @@ import numpy as np
from dios import DictOfSeries
from saqc.constants import *
-from saqc import Flagger
from saqc.core.modules.base import ModuleBase
+from saqc.core import Flags as Flagger
from saqc.lib.types import FreqString, IntegerWindow, ColumnName
diff --git a/saqc/core/modules/changepoints.py b/saqc/core/modules/changepoints.py
index 19ed26d29..7e5946cc6 100644
--- a/saqc/core/modules/changepoints.py
+++ b/saqc/core/modules/changepoints.py
@@ -8,7 +8,7 @@ from dios import DictOfSeries
from typing_extensions import Literal
from saqc.constants import *
-from saqc import Flagger
+from saqc.core import Flags as Flagger
from saqc.core.modules.base import ModuleBase
from saqc.lib.types import FreqString, IntegerWindow
diff --git a/saqc/core/modules/constants.py b/saqc/core/modules/constants.py
index 22239aa09..6787b08ed 100644
--- a/saqc/core/modules/constants.py
+++ b/saqc/core/modules/constants.py
@@ -5,7 +5,7 @@ from typing import Tuple
from dios import DictOfSeries
from saqc.constants import *
-from saqc import Flagger
+from saqc.core import Flags as Flagger
from saqc.core.modules.base import ModuleBase
from saqc.lib.types import FreqString, ColumnName
diff --git a/saqc/core/modules/curvefit.py b/saqc/core/modules/curvefit.py
index c24ce08b0..de43a906b 100644
--- a/saqc/core/modules/curvefit.py
+++ b/saqc/core/modules/curvefit.py
@@ -6,7 +6,7 @@ from dios import DictOfSeries
from typing_extensions import Literal
from saqc.constants import *
-from saqc import Flagger
+from saqc.core import Flags as Flagger
from saqc.core.modules.base import ModuleBase
diff --git a/saqc/core/modules/flagtools.py b/saqc/core/modules/flagtools.py
index 7cc2b1633..94b4748f8 100644
--- a/saqc/core/modules/flagtools.py
+++ b/saqc/core/modules/flagtools.py
@@ -7,7 +7,7 @@ import pandas as pd
from dios.dios import DictOfSeries
from typing_extensions import Literal
-from saqc import Flagger
+from saqc.core import Flags as Flagger
from saqc.constants import *
from saqc.core.modules.base import ModuleBase
from saqc.lib.types import ColumnName
diff --git a/saqc/core/modules/generic.py b/saqc/core/modules/generic.py
index da80700c3..649f5aafb 100644
--- a/saqc/core/modules/generic.py
+++ b/saqc/core/modules/generic.py
@@ -7,7 +7,8 @@ import numpy as np
import pandas as pd
from dios import DictOfSeries
-from saqc import Flagger, BAD
+from saqc.constants import *
+from saqc.core import Flags as Flagger
from saqc.core.modules.base import ModuleBase
diff --git a/saqc/core/modules/interpolation.py b/saqc/core/modules/interpolation.py
index c9aed6105..8df763b90 100644
--- a/saqc/core/modules/interpolation.py
+++ b/saqc/core/modules/interpolation.py
@@ -8,7 +8,7 @@ import pandas as pd
from dios import DictOfSeries
from typing_extensions import Literal
-from saqc import Flagger
+from saqc.core import Flags as Flagger
from saqc.constants import *
from saqc.core.modules.base import ModuleBase
from saqc.funcs.interpolation import _SUPPORTED_METHODS
diff --git a/saqc/core/modules/outliers.py b/saqc/core/modules/outliers.py
index bf8152039..d202af9b9 100644
--- a/saqc/core/modules/outliers.py
+++ b/saqc/core/modules/outliers.py
@@ -9,7 +9,7 @@ from dios import DictOfSeries
from typing_extensions import Literal
from saqc.constants import *
-from saqc import Flagger
+from saqc.core import Flags as Flagger
from saqc.core.modules.base import ModuleBase
from saqc.lib.types import IntegerWindow, FreqString, ColumnName
diff --git a/saqc/core/modules/pattern.py b/saqc/core/modules/pattern.py
index 38d083945..56db5f852 100644
--- a/saqc/core/modules/pattern.py
+++ b/saqc/core/modules/pattern.py
@@ -6,7 +6,7 @@ from typing import Sequence, Tuple
from dios import DictOfSeries
from saqc.constants import *
-from saqc import Flagger
+from saqc.core import Flags as Flagger
from saqc.core.modules.base import ModuleBase
diff --git a/saqc/core/modules/resampling.py b/saqc/core/modules/resampling.py
index 9822bbd8b..e5996987a 100644
--- a/saqc/core/modules/resampling.py
+++ b/saqc/core/modules/resampling.py
@@ -9,7 +9,7 @@ from dios import DictOfSeries
from typing_extensions import Literal
from saqc.constants import *
-from saqc import Flagger
+from saqc.core import Flags as Flagger
from saqc.core.modules.base import ModuleBase
from saqc.funcs.interpolation import _SUPPORTED_METHODS
diff --git a/saqc/core/modules/residues.py b/saqc/core/modules/residues.py
index 877323546..d0a03bac9 100644
--- a/saqc/core/modules/residues.py
+++ b/saqc/core/modules/residues.py
@@ -8,7 +8,7 @@ from dios import DictOfSeries
from typing_extensions import Literal
from saqc.constants import *
-from saqc import Flagger
+from saqc.core import Flags as Flagger
from saqc.core.modules.base import ModuleBase
diff --git a/saqc/core/modules/scores.py b/saqc/core/modules/scores.py
index a36e73d88..eafc44d09 100644
--- a/saqc/core/modules/scores.py
+++ b/saqc/core/modules/scores.py
@@ -8,7 +8,7 @@ import pandas as pd
from dios import DictOfSeries
from typing_extensions import Literal
-from saqc import Flagger
+from saqc.core import Flags as Flagger
from saqc.core.modules.base import ModuleBase
diff --git a/saqc/core/modules/tools.py b/saqc/core/modules/tools.py
index 7d3c39859..16a19bc0e 100644
--- a/saqc/core/modules/tools.py
+++ b/saqc/core/modules/tools.py
@@ -6,7 +6,7 @@ from typing import Optional, Tuple
from dios import DictOfSeries
from typing_extensions import Literal
-from saqc import Flagger
+from saqc.core import Flags as Flagger
from saqc.core.modules.base import ModuleBase
diff --git a/saqc/core/modules/transformation.py b/saqc/core/modules/transformation.py
index f85053fc5..ff94e0bbd 100644
--- a/saqc/core/modules/transformation.py
+++ b/saqc/core/modules/transformation.py
@@ -6,7 +6,7 @@ from typing import Callable, Optional, Union, Tuple
import pandas as pd
from dios import DictOfSeries
-from saqc import Flagger
+from saqc.core import Flags as Flagger
from saqc.core.modules.base import ModuleBase
diff --git a/saqc/core/reader.py b/saqc/core/reader.py
index 32d5b6985..e5aa0bce9 100644
--- a/saqc/core/reader.py
+++ b/saqc/core/reader.py
@@ -2,19 +2,17 @@
# -*- coding: utf-8 -*-
import ast
-from saqc.core.core import ColumnSelector
-
import numpy as np
-
import pandas as pd
+from saqc.core.core import ColumnSelector
from saqc.core.config import Fields as F
from saqc.core.visitor import ConfigFunctionParser
from saqc.core.lib import ConfigController
from saqc.core.register import FUNC_MAP
-
from saqc.lib.tools import isQuoted
+
COMMENT = "#"
EMPTY = "None"
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 8cd859530..c3b3945fa 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -1,6 +1,5 @@
#!/usr/bin/env python
-import logging
-from typing import Dict, Optional, Union, Tuple, List
+from typing import Dict, Optional, Union, Tuple
from typing_extensions import Literal
from functools import wraps
import dataclasses
@@ -11,8 +10,8 @@ import warnings
from saqc.constants import *
from saqc.core.lib import SaQCFunction
+from saqc.core.flags import initFlagsLike, Flags as Flagger
from saqc.lib.types import FuncReturnT
-from saqc.flagger.flags import Flagger, initFlagsLike
# NOTE:
# the global SaQC function store,
diff --git a/saqc/core/visitor.py b/saqc/core/visitor.py
index c517261e0..0fadf4878 100644
--- a/saqc/core/visitor.py
+++ b/saqc/core/visitor.py
@@ -2,7 +2,6 @@
# -*- coding: utf-8 -*-
import ast
-
import numpy as np
import pandas as pd
diff --git a/saqc/flagger/__init__.py b/saqc/flagger/__init__.py
deleted file mode 100644
index bbf082531..000000000
--- a/saqc/flagger/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from .history import History
-from .flags import Flagger, initFlagsLike
diff --git a/saqc/funcs/breaks.py b/saqc/funcs/breaks.py
index 8f10e9b72..aede9d631 100644
--- a/saqc/funcs/breaks.py
+++ b/saqc/funcs/breaks.py
@@ -20,8 +20,7 @@ from saqc.constants import *
from saqc.lib.tools import groupConsecutives
from saqc.lib.types import FreqString, ColumnName, IntegerWindow
from saqc.funcs.changepoints import assignChangePointCluster
-from saqc.core.register import register
-from saqc.flagger import Flagger
+from saqc.core import register, Flags as Flagger
@register(masking='field', module="breaks")
diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py
index 4ef620f54..7c37b9ca1 100644
--- a/saqc/funcs/changepoints.py
+++ b/saqc/funcs/changepoints.py
@@ -12,9 +12,8 @@ from typing_extensions import Literal
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core.register import register
from saqc.lib.tools import customRoller
-from saqc.flagger import Flagger
+from saqc.core import register, Flags as Flagger
from saqc.lib.types import ColumnName, FreqString, IntegerWindow
logger = logging.getLogger("SaQC")
diff --git a/saqc/funcs/constants.py b/saqc/funcs/constants.py
index 5d0b30804..a6b99a07c 100644
--- a/saqc/funcs/constants.py
+++ b/saqc/funcs/constants.py
@@ -10,8 +10,7 @@ import pandas as pd
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core.register import register
-from saqc.flagger import Flagger
+from saqc.core import register, Flags as Flagger
from saqc.lib.ts_operators import varQC
from saqc.lib.tools import customRoller, getFreqDelta
from saqc.lib.types import FreqString, ColumnName
diff --git a/saqc/funcs/curvefit.py b/saqc/funcs/curvefit.py
index f77b75346..d48d7ae4e 100644
--- a/saqc/funcs/curvefit.py
+++ b/saqc/funcs/curvefit.py
@@ -9,9 +9,8 @@ import pandas as pd
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core.register import register
+from saqc.core import register, Flags as Flagger
from saqc.lib.tools import getFreqDelta
-from saqc.flagger import Flagger
from saqc.lib.ts_operators import (
polyRollerIrregular,
polyRollerNumba,
diff --git a/saqc/funcs/drift.py b/saqc/funcs/drift.py
index 2dd2fe7c4..0892673a8 100644
--- a/saqc/funcs/drift.py
+++ b/saqc/funcs/drift.py
@@ -15,7 +15,7 @@ from scipy.spatial.distance import pdist
from saqc.constants import *
from saqc.core.register import register
-from saqc.flagger import Flagger
+from saqc.core import Flags as Flagger
from saqc.funcs.resampling import shift
from saqc.funcs.changepoints import assignChangePointCluster
from saqc.funcs.tools import drop, copy
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index 56c6a689c..db0ce930d 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -2,15 +2,12 @@
# -*- coding: utf-8 -*-
from typing import Any, Tuple, Optional, Union
from typing_extensions import Literal
-
import pandas as pd
-
from dios import DictOfSeries
from saqc.constants import *
from saqc.lib.types import *
-from saqc.core.register import register
-from saqc.flagger import Flagger
+from saqc.core import register, Flags as Flagger
import warnings
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index b8677f199..095ae1eea 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -11,9 +11,8 @@ import pandas as pd
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core.register import register
+from saqc.core import register, initFlagsLike, Flags as Flagger
from saqc.core.visitor import ENVIRONMENT
-from saqc.flagger import Flagger, initFlagsLike
import operator as op
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index a7880f4b0..9f3b985d7 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -1,18 +1,15 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-import warnings
-from typing import Tuple, Union, Optional, Any, Callable, Sequence
+from typing import Tuple, Union, Callable
from typing_extensions import Literal
-
import numpy as np
import pandas as pd
-
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core.register import register, _isflagged
-from saqc.flagger import Flagger
-from saqc.flagger.history import applyFunctionOnHistory
+from saqc.core import register, Flags as Flagger
+from saqc.core.register import _isflagged
+from saqc.core.history import applyFunctionOnHistory
from saqc.lib.ts_operators import interpolateNANs
_SUPPORTED_METHODS = Literal[
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index d8b2fcd26..e6f804be8 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -3,26 +3,18 @@
from typing import Optional, Union, Tuple, Sequence, Callable
from typing_extensions import Literal
-
+import numba
import numpy as np
import numpy.polynomial.polynomial as poly
-from scipy.optimize import curve_fit
import pandas as pd
-import numba
-
-from outliers import smirnov_grubbs
-
from dios import DictOfSeries
+from outliers import smirnov_grubbs
+from scipy.optimize import curve_fit
from saqc.constants import *
-from saqc.core.register import register
-from saqc.flagger import Flagger
-from saqc.lib.tools import (
- customRoller,
- findIndex,
- getFreqDelta
-)
+from saqc.core import register, Flags as Flagger
from saqc.lib.types import ColumnName, FreqString, IntegerWindow
+from saqc.lib.tools import customRoller, findIndex, getFreqDelta
from saqc.funcs.scores import assignKNNScore
import saqc.lib.ts_operators as ts_ops
diff --git a/saqc/funcs/pattern.py b/saqc/funcs/pattern.py
index 6562fd0d3..a51a157a4 100644
--- a/saqc/funcs/pattern.py
+++ b/saqc/funcs/pattern.py
@@ -9,8 +9,7 @@ from mlxtend.evaluate import permutation_test
from dios.dios import DictOfSeries
from saqc.constants import *
-from saqc.core.register import register
-from saqc.flagger import Flagger
+from saqc.core import register, Flags as Flagger
from saqc.lib.tools import customRoller
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 0a796d726..466bcf382 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -1,25 +1,22 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from typing import Callable, Tuple, Optional, Union, Any, Sequence
+from typing import Callable, Tuple, Optional, Union
from typing_extensions import Literal
-
-import numpy as np
import logging
-
+import numpy as np
import pandas as pd
-
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core.register import register, _isflagged
-from saqc.flagger.history import applyFunctionOnHistory
-from saqc.flagger.flags import Flagger
-from saqc.funcs.tools import copy, drop, rename
-from saqc.funcs.interpolation import interpolateIndex, _SUPPORTED_METHODS
+from saqc.core import register, Flags as Flagger
+from saqc.core.register import _isflagged
+from saqc.core.history import applyFunctionOnHistory
from saqc.lib.tools import evalFreqStr, getFreqDelta
from saqc.lib.ts_operators import shift2Freq, aggregate2Freq
-from saqc.lib.rolling import customRoller
+from saqc.funcs.tools import copy, drop, rename
+from saqc.funcs.interpolation import interpolateIndex, _SUPPORTED_METHODS
+
logger = logging.getLogger("SaQC")
diff --git a/saqc/funcs/residues.py b/saqc/funcs/residues.py
index 0b0046bea..b58c0cdf3 100644
--- a/saqc/funcs/residues.py
+++ b/saqc/funcs/residues.py
@@ -3,14 +3,11 @@
from typing import Tuple, Union, Optional, Callable
from typing_extensions import Literal
-
import numpy as np
-
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core.register import register
-from saqc.flagger import Flagger
+from saqc.core import register, Flags as Flagger
from saqc.funcs.rolling import roll
from saqc.funcs.curvefit import fitPolynomial
diff --git a/saqc/funcs/rolling.py b/saqc/funcs/rolling.py
index 6d58dfbc6..6990bb72f 100644
--- a/saqc/funcs/rolling.py
+++ b/saqc/funcs/rolling.py
@@ -2,14 +2,12 @@
# -*- coding: utf-8 -*-
from typing import Union, Callable
-
import numpy as np
import pandas as pd
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core.register import register
-from saqc.flagger import Flagger
+from saqc.core import register, Flags as Flagger
from saqc.lib.tools import getFreqDelta
diff --git a/saqc/funcs/scores.py b/saqc/funcs/scores.py
index 1f5d0456e..f1690e0fa 100644
--- a/saqc/funcs/scores.py
+++ b/saqc/funcs/scores.py
@@ -2,17 +2,14 @@
# -*- coding: utf-8 -*-
from typing import Union, Tuple, Callable, Sequence, Optional
from typing_extensions import Literal
-
import numpy as np
import pandas as pd
-
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core.register import register
-from saqc.flagger import Flagger
-from saqc.lib import ts_operators as ts_ops
+from saqc.core import register, Flags as Flagger
from saqc.lib.tools import toSequence
+import saqc.lib.ts_operators as ts_ops
@register(masking='all', module="scores")
diff --git a/saqc/funcs/tools.py b/saqc/funcs/tools.py
index c2a451b95..4ac072016 100644
--- a/saqc/funcs/tools.py
+++ b/saqc/funcs/tools.py
@@ -3,14 +3,11 @@
from typing import Optional, Tuple
from typing_extensions import Literal
-
import numpy as np
-
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core.register import register
-from saqc.flagger import Flagger
+from saqc.core import register, Flags as Flagger
from saqc.lib.tools import periodicMask
diff --git a/saqc/funcs/transformation.py b/saqc/funcs/transformation.py
index 6a176b4a9..fbda3ea6f 100644
--- a/saqc/funcs/transformation.py
+++ b/saqc/funcs/transformation.py
@@ -2,14 +2,11 @@
# -*- coding: utf-8 -*-
from typing import Optional, Callable, Tuple, Union
-
import numpy as np
import pandas as pd
-
from dios import DictOfSeries
-from saqc.core.register import register
-from saqc.flagger import Flagger
+from saqc.core import register, Flags as Flagger
@register(masking='field', module="transformation")
diff --git a/saqc/lib/plotting.py b/saqc/lib/plotting.py
index 5f79f28bb..0543110c1 100644
--- a/saqc/lib/plotting.py
+++ b/saqc/lib/plotting.py
@@ -9,7 +9,7 @@ import dios
import matplotlib.pyplot as plt
from typing import List, Dict, Optional
from saqc.constants import *
-from saqc.flagger import Flagger
+from saqc.core import Flags as Flagger
def __importHelper():
diff --git a/saqc/lib/tools.py b/saqc/lib/tools.py
index 6edfb0471..1f15df6d0 100644
--- a/saqc/lib/tools.py
+++ b/saqc/lib/tools.py
@@ -12,9 +12,9 @@ import pandas as pd
from scipy import fft
import logging
import dios
-
import collections
from scipy.cluster.hierarchy import linkage, fcluster
+
from saqc.lib.types import T
# keep this for external imports
diff --git a/saqc/lib/ts_operators.py b/saqc/lib/ts_operators.py
index 37d8253ab..b22973074 100644
--- a/saqc/lib/ts_operators.py
+++ b/saqc/lib/ts_operators.py
@@ -5,14 +5,11 @@
The module gathers all kinds of timeseries tranformations.
"""
import logging
-
import re
from typing import Union
-
import pandas as pd
import numpy as np
import numba as nb
-
from sklearn.neighbors import NearestNeighbors
from scipy.stats import iqr, median_abs_deviation
import numpy.polynomial.polynomial as poly
diff --git a/saqc/lib/types.py b/saqc/lib/types.py
index 9a437a2a9..29830a869 100644
--- a/saqc/lib/types.py
+++ b/saqc/lib/types.py
@@ -15,18 +15,18 @@ __all__ = [
from typing import TypeVar, Union, NewType
from typing_extensions import Protocol, Literal
-
import numpy as np
import pandas as pd
from dios import DictOfSeries
-from saqc import Flagger
+from saqc.core.flags import Flags
+
T = TypeVar("T")
ArrayLike = TypeVar("ArrayLike", np.ndarray, pd.Series, pd.DataFrame)
PandasLike = TypeVar("PandasLike", pd.Series, pd.DataFrame, DictOfSeries)
DiosLikeT = Union[DictOfSeries, pd.DataFrame]
-FuncReturnT = [DictOfSeries, Flagger]
+FuncReturnT = [DictOfSeries, Flags]
# we only support fixed length offsets
FreqString = NewType("FreqString", Literal["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"])
@@ -36,6 +36,7 @@ ColumnName = NewType("ColumnName", str)
IntegerWindow = NewType("IntegerWindow", int)
TimestampColumnName = TypeVar("TimestampColumnName", bound=str)
+
# needed for deeper typy hinting magic
class CurveFitter(Protocol):
def __call__(self, data: np.ndarray, *params: float) -> np.ndarray:
diff --git a/tests/common.py b/tests/common.py
index e225b6410..eddda827d 100644
--- a/tests/common.py
+++ b/tests/common.py
@@ -7,7 +7,7 @@ import pandas as pd
import dios
from saqc.constants import *
-from saqc.flagger import Flagger, initFlagsLike
+from saqc.core import initFlagsLike, Flags as Flagger
TESTNODATA = (np.nan, -9999)
diff --git a/tests/core/test_core.py b/tests/core/test_core.py
index b9773a473..c13f8a5b8 100644
--- a/tests/core/test_core.py
+++ b/tests/core/test_core.py
@@ -7,7 +7,7 @@ import numpy as np
import pandas as pd
from saqc.constants import *
-from saqc.flagger import initFlagsLike
+from saqc.core import initFlagsLike
from saqc.funcs import flagRange
from saqc.lib import plotting as splot
from saqc import SaQC, register
diff --git a/tests/core/test_creation.py b/tests/core/test_creation.py
index b16714c8e..295d2adfc 100644
--- a/tests/core/test_creation.py
+++ b/tests/core/test_creation.py
@@ -6,7 +6,7 @@ import dios
def test_init():
- from saqc import SaQC, Flagger
+ from saqc import SaQC, Flags as Flagger
arr = np.array([
[0, 1, 2],
diff --git a/tests/flagger/test_flags.py b/tests/flagger/test_flags.py
index db40b7e6c..d0d1585bc 100644
--- a/tests/flagger/test_flags.py
+++ b/tests/flagger/test_flags.py
@@ -5,7 +5,7 @@ import numpy as np
import pandas as pd
from saqc.constants import *
-from saqc.flagger.flags import Flags
+from saqc.core.flags import Flags
from tests.flagger.test_history import (
History,
diff --git a/tests/flagger/test_history.py b/tests/flagger/test_history.py
index 5a95585c0..c85eceddd 100644
--- a/tests/flagger/test_history.py
+++ b/tests/flagger/test_history.py
@@ -4,7 +4,7 @@ import pytest
import numpy as np
import pandas as pd
-from saqc.flagger.history import History
+from saqc.core.history import History
# see #GH143 combined backtrack
# (adjusted to current implementation)
diff --git a/tests/funcs/test_constants_detection.py b/tests/funcs/test_constants_detection.py
index da7b83836..6fcde58d1 100644
--- a/tests/funcs/test_constants_detection.py
+++ b/tests/funcs/test_constants_detection.py
@@ -6,7 +6,7 @@ import numpy as np
from saqc.constants import *
from saqc.funcs.constants import flagConstants, flagByVariance
-from saqc.flagger import initFlagsLike
+from saqc.core import initFlagsLike, Flags as Flagger
from tests.common import initData
diff --git a/tests/funcs/test_functions.py b/tests/funcs/test_functions.py
index 47c8ae9d2..06eef82da 100644
--- a/tests/funcs/test_functions.py
+++ b/tests/funcs/test_functions.py
@@ -4,7 +4,7 @@
import dios
from saqc.constants import *
-from saqc.flagger import initFlagsLike
+from saqc.core import initFlagsLike
from saqc.funcs.drift import flagDriftFromNorm, flagDriftFromReference, flagDriftFromScaledNorm
from saqc.funcs.outliers import flagCrossStatistic, flagRange
from saqc.funcs.flagtools import flagManual, forceFlags, clearFlags
diff --git a/tests/funcs/test_generic_config_functions.py b/tests/funcs/test_generic_config_functions.py
index fe8242cd2..5b3a28cb5 100644
--- a/tests/funcs/test_generic_config_functions.py
+++ b/tests/funcs/test_generic_config_functions.py
@@ -8,7 +8,7 @@ import pandas as pd
import dios
from saqc.constants import *
-from saqc.flagger import Flagger, initFlagsLike
+from saqc.core import initFlagsLike, Flags as Flagger
from saqc.core.visitor import ConfigFunctionParser
from saqc.core.config import Fields as F
from saqc.core.register import register
diff --git a/tests/funcs/test_harm_funcs.py b/tests/funcs/test_harm_funcs.py
index 052d5f8da..3f0140bd6 100644
--- a/tests/funcs/test_harm_funcs.py
+++ b/tests/funcs/test_harm_funcs.py
@@ -6,7 +6,7 @@ import numpy as np
import pandas as pd
import dios
-from saqc.flagger import Flagger, initFlagsLike
+from saqc.core import initFlagsLike, Flags as Flagger
from saqc.constants import BAD, UNFLAGGED
from saqc.funcs.resampling import (
linear,
diff --git a/tests/funcs/test_modelling.py b/tests/funcs/test_modelling.py
index 248c12246..95574936a 100644
--- a/tests/funcs/test_modelling.py
+++ b/tests/funcs/test_modelling.py
@@ -7,7 +7,7 @@
import dios
from saqc import BAD, UNFLAGGED
-from saqc.flagger import initFlagsLike
+from saqc.core import initFlagsLike
from saqc.funcs.tools import mask
from saqc.funcs.residues import calculatePolynomialResidues, calculateRollingResidues
diff --git a/tests/funcs/test_pattern_rec.py b/tests/funcs/test_pattern_rec.py
index b434e3c24..db3c50249 100644
--- a/tests/funcs/test_pattern_rec.py
+++ b/tests/funcs/test_pattern_rec.py
@@ -6,7 +6,7 @@ import pandas as pd
import dios
from saqc.constants import *
-from saqc.flagger import initFlagsLike
+from saqc.core import initFlagsLike
from saqc.funcs.pattern import *
from tests.common import initData
diff --git a/tests/funcs/test_proc_functions.py b/tests/funcs/test_proc_functions.py
index d9d137359..04739b40b 100644
--- a/tests/funcs/test_proc_functions.py
+++ b/tests/funcs/test_proc_functions.py
@@ -7,7 +7,7 @@
import dios
from saqc.constants import *
-from saqc.flagger import initFlagsLike
+from saqc.core import initFlagsLike
from saqc.funcs.transformation import transform
from saqc.funcs.drift import correctOffset
from saqc.funcs.interpolation import interpolateByRolling, interpolateInvalid, interpolateIndex
diff --git a/tests/funcs/test_spikes_detection.py b/tests/funcs/test_spikes_detection.py
index faa256068..727ef4157 100644
--- a/tests/funcs/test_spikes_detection.py
+++ b/tests/funcs/test_spikes_detection.py
@@ -13,7 +13,7 @@ from saqc.funcs.outliers import (
flagByGrubbs,
)
from saqc.constants import *
-from saqc.flagger import initFlagsLike
+from saqc.core import initFlagsLike
@pytest.fixture(scope="module")
diff --git a/tests/fuzzy/init.py b/tests/fuzzy/init.py
index ad93f02c6..4096823b5 100644
--- a/tests/fuzzy/init.py
+++ b/tests/fuzzy/init.py
@@ -25,7 +25,7 @@ from saqc.constants import *
from saqc.core.register import FUNC_MAP
from saqc.core.lib import SaQCFunction
from saqc.lib.types import FreqString, ColumnName, IntegerWindow
-from saqc.flagger import Flagger, initFlagsLike
+from saqc.core import initFlagsLike, Flags as Flagger
MAX_EXAMPLES = 50
# MAX_EXAMPLES = 100000
diff --git a/tests/fuzzy/test_masking.py b/tests/fuzzy/test_masking.py
index cc7637099..9567ea7f8 100644
--- a/tests/fuzzy/test_masking.py
+++ b/tests/fuzzy/test_masking.py
@@ -8,7 +8,7 @@ import pandas as pd
from hypothesis import given, settings
from saqc.constants import *
-from saqc.flagger import Flagger
+from saqc.core import Flags as Flagger
from saqc.core.register import _maskData, _unmaskData
from tests.fuzzy.init import dataFieldFlagger, MAX_EXAMPLES
--
GitLab
From 31762a1a31a58439fbc512ab5239333edebefa96 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 26 Mar 2021 00:23:31 +0100
Subject: [PATCH 093/180] prepared flagger refactoring
---
saqc/__main__.py | 10 +++++-----
saqc/core/register.py | 6 +++---
saqc/funcs/breaks.py | 4 ++--
tests/core/test_core.py | 4 ++--
tests/funcs/test_constants_detection.py | 4 ++--
tests/funcs/test_modelling.py | 12 ++++++------
6 files changed, 20 insertions(+), 20 deletions(-)
diff --git a/saqc/__main__.py b/saqc/__main__.py
index 8fbb3cb7c..7b7a3c5b0 100644
--- a/saqc/__main__.py
+++ b/saqc/__main__.py
@@ -95,12 +95,12 @@ def main(config, data, flagger, outfile, nodata, log_level, fail):
data_result, flagger_result = saqc.readConfig(config).getResult(raw=True)
if outfile:
- data_result = data_result.to_df()
- flags = flagger_result.toFrame()
- unflagged = (flags == UNFLAGGED) | flags.isna()
- flags[unflagged] = GOOD
+ data_frame = data_result.to_df()
+ flags_frame = flagger_result.toFrame()
+ unflagged = (flags_frame == UNFLAGGED) | flags_frame.isna()
+ flags_frame[unflagged] = GOOD
- fields = {"data": data_result, "flags": flags}
+ fields = {"data": data_frame, "flags": flags_frame}
out = (
pd.concat(fields.values(), axis=1, keys=fields.keys())
diff --git a/saqc/core/register.py b/saqc/core/register.py
index c3b3945fa..5d991e803 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -249,14 +249,14 @@ def _maskData(data, flagger, columns, thresh) -> Tuple[dios.DictOfSeries, dios.D
return data, mask
-def _isflagged(flags: Union[np.array, pd.Series], thresh: float) -> Union[np.array, pd.Series]:
+def _isflagged(flagscol: Union[np.array, pd.Series], thresh: float) -> Union[np.array, pd.Series]:
"""
Return a mask of flags accordingly to `thresh`. Return type is same as flags.
"""
if thresh == UNFLAGGED:
- return flags > UNFLAGGED
+ return flagscol > UNFLAGGED
- return flags >= thresh
+ return flagscol >= thresh
def _prepareFlags(flagger: Flagger, masking) -> Flagger:
diff --git a/saqc/funcs/breaks.py b/saqc/funcs/breaks.py
index aede9d631..d9cbbc6d0 100644
--- a/saqc/funcs/breaks.py
+++ b/saqc/funcs/breaks.py
@@ -124,7 +124,7 @@ def flagIsolated(
mask = data[field].isna()
- flags = pd.Series(data=0, index=mask.index, dtype=bool)
+ bools = pd.Series(data=0, index=mask.index, dtype=bool)
for srs in groupConsecutives(mask):
if np.all(~srs):
start = srs.index[0]
@@ -134,7 +134,7 @@ def flagIsolated(
if left.all():
right = mask[stop: stop + gap_window].iloc[1:]
if right.all():
- flags[start:stop] = True
+ bools[start:stop] = True
flagger[mask, field] = flag
return data, flagger
diff --git a/tests/core/test_core.py b/tests/core/test_core.py
index c13f8a5b8..a784cdbac 100644
--- a/tests/core/test_core.py
+++ b/tests/core/test_core.py
@@ -86,10 +86,10 @@ def test_dtypes(data, flags):
Test if the categorical dtype is preserved through the core functionality
"""
flagger = initFlagsLike(data)
- flags = flagger.toDios()
+ flags_raw = flagger.toDios()
var1, var2 = data.columns[:2]
- pdata, pflagger = SaQC(data, flags=flags).flagAll(var1).flagAll(var2).getResult(raw=True)
+ pdata, pflagger = SaQC(data, flags=flags_raw).flagAll(var1).flagAll(var2).getResult(raw=True)
for c in pflagger.columns:
assert pflagger[c].dtype == flagger[c].dtype
diff --git a/tests/funcs/test_constants_detection.py b/tests/funcs/test_constants_detection.py
index 6fcde58d1..d6b7a68f8 100644
--- a/tests/funcs/test_constants_detection.py
+++ b/tests/funcs/test_constants_detection.py
@@ -23,8 +23,8 @@ def test_constants_flagBasic(data):
field, *_ = data.columns
flagger = initFlagsLike(data)
data, flagger_result = flagConstants(data, field, flagger, window="15Min", thresh=0.1, flag=BAD)
- flags = flagger_result[field]
- assert np.all(flags[expected] == BAD)
+ flagscol = flagger_result[field]
+ assert np.all(flagscol[expected] == BAD)
def test_constants_flagVarianceBased(data):
diff --git a/tests/funcs/test_modelling.py b/tests/funcs/test_modelling.py
index 95574936a..de9f1efb8 100644
--- a/tests/funcs/test_modelling.py
+++ b/tests/funcs/test_modelling.py
@@ -54,20 +54,20 @@ def test_modelling_mask(dat):
common = dict(data=data, field=field, flagger=flagger, mode='periodic')
data_seasonal, flagger_seasonal = mask(**common, period_start="20:00", period_end="40:00", include_bounds=False)
- flags = flagger_seasonal[field]
- m = (20 <= flags.index.minute) & (flags.index.minute <= 40)
+ flagscol = flagger_seasonal[field]
+ m = (20 <= flagscol.index.minute) & (flagscol.index.minute <= 40)
assert all(flagger_seasonal[field][m] == UNFLAGGED)
assert all(data_seasonal[field][m].isna())
data_seasonal, flagger_seasonal = mask(**common, period_start="15:00:00", period_end="02:00:00")
- flags = flagger_seasonal[field]
- m = (15 <= flags.index.hour) & (flags.index.hour <= 2)
+ flagscol = flagger_seasonal[field]
+ m = (15 <= flagscol.index.hour) & (flagscol.index.hour <= 2)
assert all(flagger_seasonal[field][m] == UNFLAGGED)
assert all(data_seasonal[field][m].isna())
data_seasonal, flagger_seasonal = mask(**common, period_start="03T00:00:00", period_end="10T00:00:00")
- flags = flagger_seasonal[field]
- m = (3 <= flags.index.hour) & (flags.index.hour <= 10)
+ flagscol = flagger_seasonal[field]
+ m = (3 <= flagscol.index.hour) & (flagscol.index.hour <= 10)
assert all(flagger_seasonal[field][m] == UNFLAGGED)
assert all(data_seasonal[field][m].isna())
--
GitLab
From b48a445dba6133a7213a35771fc8c88cc86ce0f3 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 26 Mar 2021 00:50:49 +0100
Subject: [PATCH 094/180] extinguished flagger from code, now we have flags
---
.gitlab-ci.yml | 6 +-
saqc/__main__.py | 6 +-
saqc/core/core.py | 56 ++++++------
saqc/core/lib.py | 4 +-
saqc/core/modules/breaks.py | 8 +-
saqc/core/modules/changepoints.py | 6 +-
saqc/core/modules/constants.py | 6 +-
saqc/core/modules/curvefit.py | 4 +-
saqc/core/modules/drift.py | 14 +--
saqc/core/modules/flagtools.py | 14 +--
saqc/core/modules/generic.py | 6 +-
saqc/core/modules/interpolation.py | 8 +-
saqc/core/modules/outliers.py | 18 ++--
saqc/core/modules/pattern.py | 6 +-
saqc/core/modules/resampling.py | 16 ++--
saqc/core/modules/residues.py | 6 +-
saqc/core/modules/scores.py | 4 +-
saqc/core/modules/tools.py | 10 +-
saqc/core/modules/transformation.py | 4 +-
saqc/core/reader.py | 8 +-
saqc/core/register.py | 46 +++++-----
saqc/core/visitor.py | 2 +-
saqc/funcs/breaks.py | 24 ++---
saqc/funcs/changepoints.py | 22 ++---
saqc/funcs/constants.py | 20 ++--
saqc/funcs/curvefit.py | 14 +--
saqc/funcs/drift.py | 76 ++++++++--------
saqc/funcs/flagtools.py | 42 ++++-----
saqc/funcs/generic.py | 38 ++++----
saqc/funcs/interpolation.py | 32 +++----
saqc/funcs/outliers.py | 96 ++++++++++----------
saqc/funcs/pattern.py | 18 ++--
saqc/funcs/resampling.py | 82 ++++++++---------
saqc/funcs/residues.py | 14 +--
saqc/funcs/rolling.py | 12 +--
saqc/funcs/scores.py | 16 ++--
saqc/funcs/tools.py | 32 +++----
saqc/funcs/transformation.py | 8 +-
tests/common.py | 26 +++---
tests/core/test_core.py | 28 +++---
tests/core/test_creation.py | 4 +-
tests/{flagger => core}/test_flagger.py | 0
tests/{flagger => core}/test_flags.py | 2 +-
tests/{flagger => core}/test_history.py | 0
tests/core/test_reader.py | 8 +-
tests/flagger/__init__.py | 0
tests/funcs/test_constants_detection.py | 14 +--
tests/funcs/test_functions.py | 86 +++++++++---------
tests/funcs/test_generic_api_functions.py | 12 +--
tests/funcs/test_generic_config_functions.py | 90 +++++++++---------
tests/funcs/test_harm_funcs.py | 70 +++++++-------
tests/funcs/test_modelling.py | 48 +++++-----
tests/funcs/test_pattern_rec.py | 20 ++--
tests/funcs/test_proc_functions.py | 38 ++++----
tests/funcs/test_spikes_detection.py | 38 ++++----
tests/fuzzy/init.py | 22 ++---
tests/fuzzy/test_masking.py | 64 ++++++-------
57 files changed, 687 insertions(+), 687 deletions(-)
rename tests/{flagger => core}/test_flagger.py (100%)
rename tests/{flagger => core}/test_flags.py (99%)
rename tests/{flagger => core}/test_history.py (100%)
delete mode 100644 tests/flagger/__init__.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 490a4cf65..aeb0137bf 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,7 +24,7 @@ python37:
- schedules
image: python:3.7
script:
- - pytest tests/core tests/flagger tests/funcs
+ - pytest tests/core tests/funcs
- python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv --outfile /tmp/test.csv
@@ -34,7 +34,7 @@ python38:
except:
- schedules
script:
- - pytest tests/core tests/flagger tests/funcs
+ - pytest tests/core tests/funcs
- python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv --outfile /tmp/test.csv
@@ -46,7 +46,7 @@ coverage:
allow_failure: true
script:
- pip install pytest-cov coverage
- - pytest --cov=saqc tests/core tests/flagger tests/funcs
+ - pytest --cov=saqc tests/core tests/funcs
after_script:
- coverage xml
diff --git a/saqc/__main__.py b/saqc/__main__.py
index 7b7a3c5b0..06a30ff83 100644
--- a/saqc/__main__.py
+++ b/saqc/__main__.py
@@ -83,7 +83,7 @@ def writeData(writer_dict, df, fname):
def main(config, data, flagger, outfile, nodata, log_level, fail):
if SCHEMES[flagger] is NotImplemented:
- warnings.warn("flagger is currently not supported")
+ warnings.warn("--flagger is deprecated", DeprecationWarning)
_setupLogging(log_level)
reader, writer = setupIO(nodata)
@@ -92,11 +92,11 @@ def main(config, data, flagger, outfile, nodata, log_level, fail):
saqc = SaQC(data=data, nodata=nodata, error_policy="raise" if fail else "warn",)
- data_result, flagger_result = saqc.readConfig(config).getResult(raw=True)
+ data_result, flags_result = saqc.readConfig(config).getResult(raw=True)
if outfile:
data_frame = data_result.to_df()
- flags_frame = flagger_result.toFrame()
+ flags_frame = flags_result.toFrame()
unflagged = (flags_frame == UNFLAGGED) | flags_frame.isna()
flags_frame[unflagged] = GOOD
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 2fe5c6e11..8041f6f4f 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -18,7 +18,7 @@ import timeit
import inspect
from saqc.constants import *
-from saqc.core.flags import initFlagsLike, Flags as Flagger
+from saqc.core.flags import initFlagsLike, Flags
from saqc.core.lib import APIController, ColumnSelector
from saqc.core.register import FUNC_MAP, SaQCFunction
from saqc.core.modules import FuncModules
@@ -71,7 +71,7 @@ def _prepInput(data, flags):
if isinstance(flags.index, pd.MultiIndex) or isinstance(flags.columns, pd.MultiIndex):
raise TypeError("'flags' should not use MultiIndex")
- if isinstance(flags, (dios.DictOfSeries, pd.DataFrame, Flagger)):
+ if isinstance(flags, (dios.DictOfSeries, pd.DataFrame, Flags)):
# NOTE: only test common columns, data as well as flags could
# have more columns than the respective other.
cols = flags.columns.intersection(data.columns)
@@ -80,8 +80,8 @@ def _prepInput(data, flags):
raise ValueError(f"the index of 'flags' and 'data' missmatch in column {c}")
# this also ensures float dtype
- if not isinstance(flags, Flagger):
- flags = Flagger(flags, copy=True)
+ if not isinstance(flags, Flags):
+ flags = Flags(flags, copy=True)
return data, flags
@@ -108,30 +108,30 @@ class SaQC(FuncModules):
self._data = data
self._nodata = nodata
self._to_mask = to_mask
- self._flagger = self._initFlagger(data, flags)
+ self._flags = self._initFlags(data, flags)
self._error_policy = error_policy
# NOTE: will be filled by calls to `_wrap`
self._to_call: List[Tuple[ColumnSelector, APIController, SaQCFunction]] = []
- def _initFlagger(self, data, flagger: Union[Flagger, None]):
+ def _initFlags(self, data, flags: Union[Flags, None]):
""" Init the internal flagger object.
Ensures that all data columns are present and user passed flags from
a flags frame or an already initialised flagger are used.
"""
- if flagger is None:
+ if flags is None:
return initFlagsLike(data)
# add columns that are present in data but not in flagger
- for c in data.columns.difference(flagger.columns):
- flagger[c] = pd.Series(UNFLAGGED, index=data[c].index, dtype=float)
+ for c in data.columns.difference(flags.columns):
+ flags[c] = pd.Series(UNFLAGGED, index=data[c].index, dtype=float)
- return flagger
+ return flags
def _constructSimple(self) -> SaQC:
return SaQC(
data=dios.DictOfSeries(),
- flags=Flagger(),
+ flags=Flags(),
nodata=self._nodata,
to_mask=self._to_mask,
error_policy=self._error_policy
@@ -140,7 +140,7 @@ class SaQC(FuncModules):
def readConfig(self, fname):
from saqc.core.reader import readConfig
out = stdcopy.deepcopy(self)
- out._to_call.extend(readConfig(fname, self._flagger))
+ out._to_call.extend(readConfig(fname, self._flags))
return out
def _expandFields(self, selector: ColumnSelector, func: SaQCFunction, variables: pd.Index) -> Sequence[Tuple[ColumnSelector, SaQCFunction]]:
@@ -166,15 +166,15 @@ class SaQC(FuncModules):
# NOTE: It would be nicer to separate the plotting into an own
# method instead of intermingling it with the computation
- data, flagger = self._data, self._flagger
+ data, flags = self._data, self._flags
for selector, control, function in self._to_call:
- for sel, func in self._expandFields(selector, function, data.columns.union(flagger.columns)):
+ for sel, func in self._expandFields(selector, function, data.columns.union(flags.columns)):
logger.debug(f"processing: {sel.field}, {func.name}, {func.keywords}")
t0 = timeit.default_timer()
try:
- data_result, flagger_result = _saqcCallFunc(sel, control, func, data, flagger)
+ data_result, flags_result = _saqcCallFunc(sel, control, func, data, flags)
except Exception as e:
t1 = timeit.default_timer()
logger.debug(f"{func.name} failed after {t1 - t0} sec")
@@ -188,23 +188,23 @@ class SaQC(FuncModules):
plotHook(
data_old=data,
data_new=data_result,
- flagger_old=flagger,
- flagger_new=flagger_result,
+ flagger_old=flags,
+ flagger_new=flags_result,
sources=[],
targets=[sel.field],
plot_name=func.name,
)
data = data_result
- flagger = flagger_result
+ flags = flags_result
if any([control.plot for _, control, _ in self._to_call]):
- plotAllHook(data, flagger)
+ plotAllHook(data, flags)
# This is way faster for big datasets, than to throw everything in the constructor.
# Simply because of _initFlagger -> merge() -> mergeDios() over all columns.
new = self._constructSimple()
- new._flagger, new._data = flagger, data
+ new._flags, new._data = flags, data
return new
def getResult(self, raw=False):
@@ -217,12 +217,12 @@ class SaQC(FuncModules):
"""
realization = self.evaluate()
- data, flagger = realization._data, realization._flagger
+ data, flags = realization._data, realization._flags
if raw:
- return data, flagger
+ return data, flags
- return data.to_df(), flagger.toFrame()
+ return data.to_df(), flags.toFrame()
def _wrap(self, func: SaQCFunction):
@@ -267,26 +267,26 @@ class SaQC(FuncModules):
return stdcopy.copy(self)
-def _saqcCallFunc(locator, controller, function, data, flagger):
+def _saqcCallFunc(locator, controller, function, data, flags):
# NOTE:
# We assure that all columns in data have an equivalent column in flags,
# we might have more flagger columns though
- assert data.columns.difference(flagger.columns).empty
+ assert data.columns.difference(flags.columns).empty
field = locator.field
target = locator.target
if (target != field) and (locator.regex is False):
- data, flagger = copy(data, field, flagger, target)
+ data, flags = copy(data, field, flags, target)
field = target
- data_result, flagger_result = function(data, field, flagger)
+ data_result, flags_result = function(data, field, flags)
# we check the passed function-kwargs after the actual call, because now "hard" errors would already have been
# raised (Eg. `TypeError: got multiple values for argument 'data'`, when the user pass data=...)
_warnForUnusedKwargs(function)
- return data_result, flagger_result
+ return data_result, flags_result
def _warnForUnusedKwargs(func):
diff --git a/saqc/core/lib.py b/saqc/core/lib.py
index 2236e3b63..24fb29633 100644
--- a/saqc/core/lib.py
+++ b/saqc/core/lib.py
@@ -50,9 +50,9 @@ class SaQCFunction:
**{**self.keywords, **keywords}
)
- def __call__(self, data, field, flagger, *args, **keywords):
+ def __call__(self, data, field, flags, *args, **keywords):
keywords = {**self.keywords, **keywords}
- return self.func(data, field, flagger, *self.args, *args, **keywords)
+ return self.func(data, field, flags, *self.args, *args, **keywords)
def errorMessage(self) -> str:
return f"function: {self.name}\narguments: {self.args}\nkeywords: {self.keywords}"
diff --git a/saqc/core/modules/breaks.py b/saqc/core/modules/breaks.py
index 6fab21ff3..49826f4c1 100644
--- a/saqc/core/modules/breaks.py
+++ b/saqc/core/modules/breaks.py
@@ -7,7 +7,7 @@ from dios import DictOfSeries
from saqc.constants import *
from saqc.core.modules.base import ModuleBase
-from saqc.core import Flags as Flagger
+from saqc.core import Flags
from saqc.lib.types import FreqString, IntegerWindow, ColumnName
@@ -19,7 +19,7 @@ class Breaks(ModuleBase):
nodata: float = np.nan,
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagMissing", locals())
def flagIsolated(
@@ -29,7 +29,7 @@ class Breaks(ModuleBase):
group_window: FreqString,
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagIsolated", locals())
def flagJumps(
@@ -40,5 +40,5 @@ class Breaks(ModuleBase):
min_periods: IntegerWindow = 1,
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagJumps", locals())
diff --git a/saqc/core/modules/changepoints.py b/saqc/core/modules/changepoints.py
index 7e5946cc6..bab2fe897 100644
--- a/saqc/core/modules/changepoints.py
+++ b/saqc/core/modules/changepoints.py
@@ -8,7 +8,7 @@ from dios import DictOfSeries
from typing_extensions import Literal
from saqc.constants import *
-from saqc.core import Flags as Flagger
+from saqc.core import Flags
from saqc.core.modules.base import ModuleBase
from saqc.lib.types import FreqString, IntegerWindow
@@ -29,7 +29,7 @@ class ChangePoints(ModuleBase):
reduce_func: Callable[[np.ndarray, np.ndarray], int] = lambda x, _: x.argmax(),
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagChangePoints", locals())
def assignChangePointCluster(
@@ -49,5 +49,5 @@ class ChangePoints(ModuleBase):
assign_cluster: bool = True,
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("assignChangePointCluster", locals())
diff --git a/saqc/core/modules/constants.py b/saqc/core/modules/constants.py
index 6787b08ed..cf6e9851c 100644
--- a/saqc/core/modules/constants.py
+++ b/saqc/core/modules/constants.py
@@ -5,7 +5,7 @@ from typing import Tuple
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core import Flags as Flagger
+from saqc.core import Flags
from saqc.core.modules.base import ModuleBase
from saqc.lib.types import FreqString, ColumnName
@@ -21,7 +21,7 @@ class Constants(ModuleBase):
max_consec_missing: int = None,
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagByVariance", locals())
def flagConstants(
@@ -31,5 +31,5 @@ class Constants(ModuleBase):
window: FreqString,
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagConstants", locals())
diff --git a/saqc/core/modules/curvefit.py b/saqc/core/modules/curvefit.py
index de43a906b..edb9aa75b 100644
--- a/saqc/core/modules/curvefit.py
+++ b/saqc/core/modules/curvefit.py
@@ -6,7 +6,7 @@ from dios import DictOfSeries
from typing_extensions import Literal
from saqc.constants import *
-from saqc.core import Flags as Flagger
+from saqc.core import Flags
from saqc.core.modules.base import ModuleBase
@@ -22,5 +22,5 @@ class Curvefit(ModuleBase):
return_residues: bool = False,
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("fitPolynomial", locals())
diff --git a/saqc/core/modules/drift.py b/saqc/core/modules/drift.py
index e063e62f3..0616dd53d 100644
--- a/saqc/core/modules/drift.py
+++ b/saqc/core/modules/drift.py
@@ -8,7 +8,7 @@ from scipy.spatial.distance import pdist
from saqc.constants import *
from saqc.core.modules.base import ModuleBase
-from saqc.funcs import LinkageString, DictOfSeries, Flagger
+from saqc.funcs import LinkageString, DictOfSeries, Flags
from saqc.lib.types import ColumnName, FreqString, CurveFitter
@@ -24,7 +24,7 @@ class Drift(ModuleBase):
linkage_method: LinkageString = "single",
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagDriftFromNorm", locals())
def flagDriftFromReference(
@@ -36,7 +36,7 @@ class Drift(ModuleBase):
metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagDriftFromReference", locals())
def flagDriftFromScaledNorm(
@@ -51,7 +51,7 @@ class Drift(ModuleBase):
linkage_method: LinkageString = "single",
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagDriftFromScaledNorm", locals())
def correctExponentialDrift(
@@ -62,7 +62,7 @@ class Drift(ModuleBase):
flag_maint_period: bool = False,
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("correctExponentialDrift", locals())
def correctRegimeAnomaly(
@@ -73,7 +73,7 @@ class Drift(ModuleBase):
regime_transmission: Optional[FreqString] = None,
x_date: bool = False,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("correctRegimeAnomaly", locals())
def correctOffset(
@@ -85,5 +85,5 @@ class Drift(ModuleBase):
min_periods: int,
regime_transmission: Optional[FreqString] = None,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("correctOffset", locals())
diff --git a/saqc/core/modules/flagtools.py b/saqc/core/modules/flagtools.py
index 94b4748f8..68bd70364 100644
--- a/saqc/core/modules/flagtools.py
+++ b/saqc/core/modules/flagtools.py
@@ -7,7 +7,7 @@ import pandas as pd
from dios.dios import DictOfSeries
from typing_extensions import Literal
-from saqc.core import Flags as Flagger
+from saqc.core import Flags
from saqc.constants import *
from saqc.core.modules.base import ModuleBase
from saqc.lib.types import ColumnName
@@ -15,15 +15,15 @@ from saqc.lib.types import ColumnName
class FlagTools(ModuleBase):
- def clearFlags(self, field: ColumnName, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+ def clearFlags(self, field: ColumnName, **kwargs) -> Tuple[DictOfSeries, Flags]:
return self.defer("clearFlags", locals())
def forceFlags(
self, field: ColumnName, flag: float = BAD, **kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("forceFlags", locals())
- def flagDummy(self, field: ColumnName, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+ def flagDummy(self, field: ColumnName, **kwargs) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagDummy", locals())
def flagForceFail(self, field: ColumnName, **kwargs):
@@ -31,10 +31,10 @@ class FlagTools(ModuleBase):
def flagUnflagged(
self, field: ColumnName, flag: float = BAD, **kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagUnflagged", locals())
- def flagGood(self, field: ColumnName, flag=BAD, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+ def flagGood(self, field: ColumnName, flag=BAD, **kwargs) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagGood", locals())
def flagManual(
@@ -44,5 +44,5 @@ class FlagTools(ModuleBase):
method: Literal["plain", "ontime", "left-open", "right-open"] = 'plain',
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagManual", locals())
diff --git a/saqc/core/modules/generic.py b/saqc/core/modules/generic.py
index 649f5aafb..87cde4f93 100644
--- a/saqc/core/modules/generic.py
+++ b/saqc/core/modules/generic.py
@@ -8,7 +8,7 @@ import pandas as pd
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core import Flags as Flagger
+from saqc.core import Flags
from saqc.core.modules.base import ModuleBase
@@ -20,7 +20,7 @@ class Generic(ModuleBase):
func: Callable[[pd.Series], pd.Series],
nodata: float = np.nan,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("process", locals())
def flag(
@@ -30,5 +30,5 @@ class Generic(ModuleBase):
nodata: float = np.nan,
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flag", locals())
diff --git a/saqc/core/modules/interpolation.py b/saqc/core/modules/interpolation.py
index 8df763b90..0b31e4618 100644
--- a/saqc/core/modules/interpolation.py
+++ b/saqc/core/modules/interpolation.py
@@ -8,7 +8,7 @@ import pandas as pd
from dios import DictOfSeries
from typing_extensions import Literal
-from saqc.core import Flags as Flagger
+from saqc.core import Flags
from saqc.constants import *
from saqc.core.modules.base import ModuleBase
from saqc.funcs.interpolation import _SUPPORTED_METHODS
@@ -24,7 +24,7 @@ class Interpolation(ModuleBase):
min_periods: int = 0,
flag: float = UNFLAGGED,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("interpolateByRolling", locals())
def interpolateInvalid(
@@ -36,7 +36,7 @@ class Interpolation(ModuleBase):
downgrade_interpolation: bool = False,
flag: float = UNFLAGGED,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("interpolateInvalid", locals())
def interpolateIndex(
@@ -48,6 +48,6 @@ class Interpolation(ModuleBase):
inter_limit: int = 2,
downgrade_interpolation: bool = False,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("interpolateIndex", locals())
diff --git a/saqc/core/modules/outliers.py b/saqc/core/modules/outliers.py
index d202af9b9..40737dd6d 100644
--- a/saqc/core/modules/outliers.py
+++ b/saqc/core/modules/outliers.py
@@ -9,7 +9,7 @@ from dios import DictOfSeries
from typing_extensions import Literal
from saqc.constants import *
-from saqc.core import Flags as Flagger
+from saqc.core import Flags
from saqc.core.modules.base import ModuleBase
from saqc.lib.types import IntegerWindow, FreqString, ColumnName
@@ -25,7 +25,7 @@ class Outliers(ModuleBase):
alpha: float = 0.05,
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagByStray", locals())
def flagMVScores(
@@ -46,7 +46,7 @@ class Outliers(ModuleBase):
reduction_min_periods: int = 1,
flag: float = BAD,
**kwargs,
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagMVScores", locals())
def flagRaise(
@@ -62,7 +62,7 @@ class Outliers(ModuleBase):
numba_boost: bool = True, # TODO: rm, not a user decision
flag: float = BAD,
**kwargs,
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagRaise", locals())
def flagMAD(
@@ -72,7 +72,7 @@ class Outliers(ModuleBase):
z: float = 3.5,
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagMAD", locals())
def flagOffset(
@@ -85,7 +85,7 @@ class Outliers(ModuleBase):
numba_kickin: int = 200000, # TODO: rm, not a user decision
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagOffset", locals())
def flagByGrubbs(
@@ -97,7 +97,7 @@ class Outliers(ModuleBase):
check_lagged: bool = False,
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagByGrubbs", locals())
def flagRange(
@@ -107,7 +107,7 @@ class Outliers(ModuleBase):
max: float = np.inf,
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagRange", locals())
def flagCrossStatistic(
@@ -118,5 +118,5 @@ class Outliers(ModuleBase):
cross_stat: Literal["modZscore", "Zscore"] = "modZscore",
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagCrossStatistic", locals())
diff --git a/saqc/core/modules/pattern.py b/saqc/core/modules/pattern.py
index 56db5f852..16ab2949a 100644
--- a/saqc/core/modules/pattern.py
+++ b/saqc/core/modules/pattern.py
@@ -6,7 +6,7 @@ from typing import Sequence, Tuple
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core import Flags as Flagger
+from saqc.core import Flags
from saqc.core.modules.base import ModuleBase
@@ -20,7 +20,7 @@ class Pattern(ModuleBase):
waveform: str = "mexh",
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagPatternByDTW", locals())
def flagPatternByWavelet(
@@ -31,5 +31,5 @@ class Pattern(ModuleBase):
normalize: bool = True,
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("flagPatternByWavelet", locals())
diff --git a/saqc/core/modules/resampling.py b/saqc/core/modules/resampling.py
index e5996987a..3100d79e0 100644
--- a/saqc/core/modules/resampling.py
+++ b/saqc/core/modules/resampling.py
@@ -9,7 +9,7 @@ from dios import DictOfSeries
from typing_extensions import Literal
from saqc.constants import *
-from saqc.core import Flags as Flagger
+from saqc.core import Flags
from saqc.core.modules.base import ModuleBase
from saqc.funcs.interpolation import _SUPPORTED_METHODS
@@ -25,7 +25,7 @@ class Resampling(ModuleBase):
method: Literal["fagg", "bagg", "nagg"] = "nagg",
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("aggregate", locals())
def linear(
@@ -33,7 +33,7 @@ class Resampling(ModuleBase):
field: str,
freq: str,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("linear", locals())
def interpolate(
@@ -43,7 +43,7 @@ class Resampling(ModuleBase):
method: _SUPPORTED_METHODS,
order: int = 1,
**kwargs,
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("interpolate", locals())
def mapToOriginal(
@@ -55,7 +55,7 @@ class Resampling(ModuleBase):
"inverse_interpolation"
],
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("mapToOriginal", locals())
def shift(
@@ -65,7 +65,7 @@ class Resampling(ModuleBase):
method: Literal["fshift", "bshift", "nshift"] = "nshift",
freq_check: Optional[Literal["check", "auto"]] = None, # TODO: not a user decision
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("shift", locals())
def resample(
@@ -81,7 +81,7 @@ class Resampling(ModuleBase):
flag_agg_func: Callable[[pd.Series], float] = max,
freq_check: Optional[Literal["check", "auto"]] = None,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("resample", locals())
def reindexFlags(
@@ -94,5 +94,5 @@ class Resampling(ModuleBase):
source: str,
freq: Optional[str] = None,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("reindexFlags", locals())
diff --git a/saqc/core/modules/residues.py b/saqc/core/modules/residues.py
index d0a03bac9..dc8fd8bb3 100644
--- a/saqc/core/modules/residues.py
+++ b/saqc/core/modules/residues.py
@@ -8,7 +8,7 @@ from dios import DictOfSeries
from typing_extensions import Literal
from saqc.constants import *
-from saqc.core import Flags as Flagger
+from saqc.core import Flags
from saqc.core.modules.base import ModuleBase
@@ -24,7 +24,7 @@ class Residues(ModuleBase):
min_periods: Optional[int] = 0,
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("calculatePolynomialResidues", locals())
def calculateRollingResidues(
@@ -37,5 +37,5 @@ class Residues(ModuleBase):
center: bool = True,
flag: float = BAD,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("calculateRollingResidues", locals())
diff --git a/saqc/core/modules/scores.py b/saqc/core/modules/scores.py
index eafc44d09..7b52179c1 100644
--- a/saqc/core/modules/scores.py
+++ b/saqc/core/modules/scores.py
@@ -8,7 +8,7 @@ import pandas as pd
from dios import DictOfSeries
from typing_extensions import Literal
-from saqc.core import Flags as Flagger
+from saqc.core import Flags
from saqc.core.modules.base import ModuleBase
@@ -29,5 +29,5 @@ class Scores(ModuleBase):
metric: str = 'minkowski',
p: int = 2,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("assignKNNScore", locals())
diff --git a/saqc/core/modules/tools.py b/saqc/core/modules/tools.py
index 16a19bc0e..70469b1f3 100644
--- a/saqc/core/modules/tools.py
+++ b/saqc/core/modules/tools.py
@@ -6,18 +6,18 @@ from typing import Optional, Tuple
from dios import DictOfSeries
from typing_extensions import Literal
-from saqc.core import Flags as Flagger
+from saqc.core import Flags
from saqc.core.modules.base import ModuleBase
class Tools(ModuleBase):
- def copy(self, field: str, new_field: str, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+ def copy(self, field: str, new_field: str, **kwargs) -> Tuple[DictOfSeries, Flags]:
return self.defer("copy", locals())
- def drop(self, field: str, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+ def drop(self, field: str, **kwargs) -> Tuple[DictOfSeries, Flags]:
return self.defer("drop", locals())
- def rename(self, field: str, new_name: str, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+ def rename(self, field: str, new_name: str, **kwargs) -> Tuple[DictOfSeries, Flags]:
return self.defer("rename", locals())
def mask(
@@ -29,5 +29,5 @@ class Tools(ModuleBase):
period_end: Optional[str]=None,
include_bounds: bool=True,
**kwargs,
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("mask", locals())
diff --git a/saqc/core/modules/transformation.py b/saqc/core/modules/transformation.py
index ff94e0bbd..9fcddac47 100644
--- a/saqc/core/modules/transformation.py
+++ b/saqc/core/modules/transformation.py
@@ -6,7 +6,7 @@ from typing import Callable, Optional, Union, Tuple
import pandas as pd
from dios import DictOfSeries
-from saqc.core import Flags as Flagger
+from saqc.core import Flags
from saqc.core.modules.base import ModuleBase
@@ -18,5 +18,5 @@ class Transformation(ModuleBase):
func: Callable[[pd.Series], pd.Series],
partition_freq: Optional[Union[float, str]] = None,
**kwargs
- ) -> Tuple[DictOfSeries, Flagger]:
+ ) -> Tuple[DictOfSeries, Flags]:
return self.defer("transform", locals())
diff --git a/saqc/core/reader.py b/saqc/core/reader.py
index e5aa0bce9..9be731b06 100644
--- a/saqc/core/reader.py
+++ b/saqc/core/reader.py
@@ -56,7 +56,7 @@ def _injectOptionalColumns(df):
return df
-def _parseConfig(df, flagger):
+def _parseConfig(df, flags):
funcs = []
for lineno, (_, target, expr, plot) in enumerate(df.itertuples()):
if target == "None" or pd.isnull(target) or pd.isnull(expr):
@@ -68,7 +68,7 @@ def _parseConfig(df, flagger):
target = target[1:-1]
tree = ast.parse(expr, mode="eval")
- func_name, kwargs = ConfigFunctionParser(flagger).parse(tree.body)
+ func_name, kwargs = ConfigFunctionParser(flags).parse(tree.body)
func = FUNC_MAP[func_name]
selector = ColumnSelector(
@@ -89,7 +89,7 @@ def _parseConfig(df, flagger):
return funcs
-def readConfig(fname, flagger):
+def readConfig(fname, flags):
df = pd.read_csv(
fname,
sep=r"\s*;\s*",
@@ -108,4 +108,4 @@ def readConfig(fname, flagger):
df[F.TEST] = df[F.TEST].replace(r"^\s*$", np.nan, regex=True)
df[F.PLOT] = df[F.PLOT].replace({"False": "", EMPTY: "", np.nan: ""})
df = df.astype({F.PLOT: bool})
- return _parseConfig(df, flagger)
+ return _parseConfig(df, flags)
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 5d991e803..b00f353d0 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -10,7 +10,7 @@ import warnings
from saqc.constants import *
from saqc.core.lib import SaQCFunction
-from saqc.core.flags import initFlagsLike, Flags as Flagger
+from saqc.core.flags import initFlagsLike, Flags
from saqc.lib.types import FuncReturnT
# NOTE:
@@ -26,7 +26,7 @@ class CallState:
func: callable
data: dios.DictOfSeries
- flagger: Flagger
+ flags: Flags
field: str
args: tuple
@@ -48,8 +48,8 @@ def register(masking: MaskingStrT = "all", module: Optional[str] = None):
# executed if a register-decorated function is called,
# nevertheless if it is called plain or via `SaQC.func`.
@wraps(func)
- def callWrapper(data, field, flagger, *args, **kwargs):
- args = data, field, flagger, *args
+ def callWrapper(data, field, flags, *args, **kwargs):
+ args = data, field, flags, *args
args, kwargs, old_state = _preCall(func, args, kwargs, masking, func_name)
result = func(*args, **kwargs)
return _postCall(result, old_state)
@@ -99,25 +99,25 @@ def _preCall(func: callable, args: tuple, kwargs: dict, masking: MaskingStrT, fn
mthresh = _getMaskingThresh(masking, kwargs, fname)
kwargs['to_mask'] = mthresh
- data, field, flagger, *args = args
+ data, field, flags, *args = args
# handle data - masking
columns = _getMaskingColumns(data, field, masking)
- masked_data, mask = _maskData(data, flagger, columns, mthresh)
+ masked_data, mask = _maskData(data, flags, columns, mthresh)
# store current state
state = CallState(
func=func,
- data=data, flagger=flagger, field=field,
+ data=data, flags=flags, field=field,
args=args, kwargs=kwargs,
masking=masking, mthresh=mthresh,
mask=mask
)
# handle flags - clearing
- prepped_flagger = _prepareFlags(flagger, masking)
+ prepped_flags = _prepareFlags(flags, masking)
- args = masked_data, field, prepped_flagger, *args
+ args = masked_data, field, prepped_flags, *args
return args, kwargs, state
@@ -140,10 +140,10 @@ def _postCall(result, old_state: CallState) -> FuncReturnT:
-------
data, flagger : dios.DictOfSeries, saqc.flagger.Flagger
"""
- data, flagger = result
- flagger = _restoreFlags(flagger, old_state)
+ data, flags = result
+ flags = _restoreFlags(flags, old_state)
data = _unmaskData(data, old_state)
- return data, flagger
+ return data, flags
def _getMaskingColumns(data: dios.DictOfSeries, field: str, masking: MaskingStrT):
@@ -220,7 +220,7 @@ def _getMaskingThresh(masking, kwargs, fname):
# TODO: this is heavily undertested
-def _maskData(data, flagger, columns, thresh) -> Tuple[dios.DictOfSeries, dios.DictOfSeries]:
+def _maskData(data, flags, columns, thresh) -> Tuple[dios.DictOfSeries, dios.DictOfSeries]:
"""
Mask data with Nans by flags worse that a threshold and according to ``masking`` keyword
from the functions decorator.
@@ -237,7 +237,7 @@ def _maskData(data, flagger, columns, thresh) -> Tuple[dios.DictOfSeries, dios.D
# we use numpy here because it is faster
for c in columns:
- col_mask = _isflagged(flagger[c].to_numpy(), thresh)
+ col_mask = _isflagged(flags[c].to_numpy(), thresh)
if any(col_mask):
col_data = data[c].to_numpy(dtype=np.float64)
@@ -259,7 +259,7 @@ def _isflagged(flagscol: Union[np.array, pd.Series], thresh: float) -> Union[np.
return flagscol >= thresh
-def _prepareFlags(flagger: Flagger, masking) -> Flagger:
+def _prepareFlags(flags: Flags, masking) -> Flags:
"""
Prepare flags before each call. Always returns a copy.
@@ -269,27 +269,27 @@ def _prepareFlags(flagger: Flagger, masking) -> Flagger:
"""
# Either the index or the columns itself changed
if masking == 'none':
- return flagger.copy()
+ return flags.copy()
- return initFlagsLike(flagger, initial_value=UNTOUCHED)
+ return initFlagsLike(flags, initial_value=UNTOUCHED)
-def _restoreFlags(flagger: Flagger, old_state: CallState):
+def _restoreFlags(flags: Flags, old_state: CallState):
if old_state.masking == 'none':
- return flagger
+ return flags
- columns = flagger.columns
+ columns = flags.columns
# take field column and all possibly newly added columns
if old_state.masking == 'field':
- columns = columns.difference(old_state.flagger.columns)
+ columns = columns.difference(old_state.flags.columns)
columns = columns.append(pd.Index([old_state.field]))
- out = old_state.flagger.copy()
+ out = old_state.flags.copy()
for c in columns:
# this implicitly squash the new-flagger history (RHS) to a single column, which than is appended to
# the old history (LHS). The new-flagger history possibly consist of multiple columns, one for each
# time flags was set to the flagger.
- out[c] = flagger[c]
+ out[c] = flags[c]
return out
diff --git a/saqc/core/visitor.py b/saqc/core/visitor.py
index 0fadf4878..7d7203fa0 100644
--- a/saqc/core/visitor.py
+++ b/saqc/core/visitor.py
@@ -137,7 +137,7 @@ class ConfigFunctionParser(ast.NodeVisitor):
ast.Attribute
)
- def __init__(self, flagger):
+ def __init__(self, flags):
self.kwargs = {}
self.environment = {
diff --git a/saqc/funcs/breaks.py b/saqc/funcs/breaks.py
index d9cbbc6d0..f3ad9eeb2 100644
--- a/saqc/funcs/breaks.py
+++ b/saqc/funcs/breaks.py
@@ -20,18 +20,18 @@ from saqc.constants import *
from saqc.lib.tools import groupConsecutives
from saqc.lib.types import FreqString, ColumnName, IntegerWindow
from saqc.funcs.changepoints import assignChangePointCluster
-from saqc.core import register, Flags as Flagger
+from saqc.core import register, Flags
@register(masking='field', module="breaks")
def flagMissing(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
nodata: float = np.nan,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
The function flags all values indicating missing data.
@@ -62,20 +62,20 @@ def flagMissing(
else:
mask = datacol == nodata
- flagger[mask, field] = flag
- return data, flagger
+ flags[mask, field] = flag
+ return data, flags
@register(masking='field', module="breaks")
def flagIsolated(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
gap_window: FreqString,
group_window: FreqString,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
The function flags arbitrary large groups of values, if they are surrounded by sufficiently
large data gaps.
@@ -136,21 +136,21 @@ def flagIsolated(
if right.all():
bools[start:stop] = True
- flagger[mask, field] = flag
- return data, flagger
+ flags[mask, field] = flag
+ return data, flags
@register(masking='field', module="breaks")
def flagJumps(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
thresh: float,
winsz: FreqString,
min_periods: IntegerWindow = 1,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Flag datapoints, where the mean of the values significantly changes (where the value course "jumps").
@@ -174,7 +174,7 @@ def flagJumps(
flag to set.
"""
return assignChangePointCluster(
- data, field, flagger,
+ data, field, flags,
stat_func=lambda x, y: np.abs(np.mean(x) - np.mean(y)),
thresh_func=lambda x, y: thresh,
bwd_window=winsz,
diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py
index 7c37b9ca1..83439157e 100644
--- a/saqc/funcs/changepoints.py
+++ b/saqc/funcs/changepoints.py
@@ -13,7 +13,7 @@ from dios import DictOfSeries
from saqc.constants import *
from saqc.lib.tools import customRoller
-from saqc.core import register, Flags as Flagger
+from saqc.core import register, Flags
from saqc.lib.types import ColumnName, FreqString, IntegerWindow
logger = logging.getLogger("SaQC")
@@ -21,7 +21,7 @@ logger = logging.getLogger("SaQC")
@register(masking='field', module="changepoints")
def flagChangePoints(
- data: DictOfSeries, field: str, flagger: Flagger,
+ data: DictOfSeries, field: str, flags: Flags,
stat_func: Callable[[np.ndarray, np.ndarray], float],
thresh_func: Callable[[np.ndarray, np.ndarray], float],
bwd_window: FreqString,
@@ -34,7 +34,7 @@ def flagChangePoints(
reduce_func: Callable[[np.ndarray, np.ndarray], int] = lambda x, _: x.argmax(),
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Flag datapoints, where the parametrization of the process, the data is assumed to generate by, significantly
changes.
@@ -88,7 +88,7 @@ def flagChangePoints(
return assignChangePointCluster(
data,
field,
- flagger,
+ flags,
stat_func=stat_func,
thresh_func=thresh_func,
bwd_window=bwd_window,
@@ -109,7 +109,7 @@ def flagChangePoints(
@register(masking='field', module="changepoints")
def assignChangePointCluster(
- data: DictOfSeries, field: str, flagger: Flagger,
+ data: DictOfSeries, field: str, flags: Flags,
stat_func: Callable[[np.array, np.array], float],
thresh_func: Callable[[np.array, np.array], float],
bwd_window: str,
@@ -125,7 +125,7 @@ def assignChangePointCluster(
assign_cluster: bool = True,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Assigns label to the data, aiming to reflect continous regimes of the processes the data is assumed to be
generated by.
@@ -233,8 +233,8 @@ def assignChangePointCluster(
residues = pd.Series(np.nan, index=data[field].index)
residues[masked_index] = stat_arr
data[field] = residues
- flagger[:, field] = UNFLAGGED
- return data, flagger
+ flags[:, field] = UNFLAGGED
+ return data, flags
det_index = masked_index[result_arr]
detected = pd.Series(True, index=det_index)
@@ -253,11 +253,11 @@ def assignChangePointCluster(
# (better to start cluster labels with number one)
cluster += 1
data[field] = cluster
- flagger[:, field] = UNFLAGGED
+ flags[:, field] = UNFLAGGED
if flag_changepoints:
- flagger[det_index, field] = flag
- return data, flagger
+ flags[det_index, field] = flag
+ return data, flags
@numba.jit(parallel=True, nopython=True)
diff --git a/saqc/funcs/constants.py b/saqc/funcs/constants.py
index a6b99a07c..3791eaabf 100644
--- a/saqc/funcs/constants.py
+++ b/saqc/funcs/constants.py
@@ -10,7 +10,7 @@ import pandas as pd
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core import register, Flags as Flagger
+from saqc.core import register, Flags
from saqc.lib.ts_operators import varQC
from saqc.lib.tools import customRoller, getFreqDelta
from saqc.lib.types import FreqString, ColumnName
@@ -20,12 +20,12 @@ from saqc.lib.types import FreqString, ColumnName
def flagConstants(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
thresh: float,
window: FreqString,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
This functions flags plateaus/series of constant values of length `window` if
their maximum total change is smaller than thresh.
@@ -76,22 +76,22 @@ def flagConstants(
m2 = r.max() - r.min() <= thresh
mask = m1 | m2
- flagger[mask, field] = flag
- return data, flagger
+ flags[mask, field] = flag
+ return data, flags
@register(masking='field', module="constants")
def flagByVariance(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
window: FreqString = "12h",
thresh: float = 0.0005,
max_missing: int = None,
max_consec_missing: int = None,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Function flags plateaus/series of constant values. Any interval of values y(t),..y(t+n) is flagged, if:
@@ -153,12 +153,12 @@ def flagByVariance(
# are there any candidates for beeing flagged plateau-ish
if plateaus.sum() == 0:
- return data, flagger
+ return data, flags
plateaus.fillna(method="bfill", limit=min_periods - 1, inplace=True)
# result:
plateaus = (plateaus[plateaus == 1.0]).index
- flagger[plateaus, field] = flag
- return data, flagger
+ flags[plateaus, field] = flag
+ return data, flags
diff --git a/saqc/funcs/curvefit.py b/saqc/funcs/curvefit.py
index d48d7ae4e..3465e07d1 100644
--- a/saqc/funcs/curvefit.py
+++ b/saqc/funcs/curvefit.py
@@ -9,7 +9,7 @@ import pandas as pd
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core import register, Flags as Flagger
+from saqc.core import register, Flags
from saqc.lib.tools import getFreqDelta
from saqc.lib.ts_operators import (
polyRollerIrregular,
@@ -24,7 +24,7 @@ from saqc.lib.ts_operators import (
def fitPolynomial(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
winsz: Union[int, str],
polydeg: int,
numba: Literal[True, False, "auto"] = "auto",
@@ -33,7 +33,7 @@ def fitPolynomial(
return_residues: bool = False,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Function fits a polynomial model to the data and returns the fitted data curve.
@@ -112,7 +112,7 @@ def fitPolynomial(
"""
# TODO: some (rater large) parts are functional similar to saqc.funcs.rolling.roll
if data[field].empty:
- return data, flagger
+ return data, flags
data = data.copy()
to_fit = data[field]
regular = getFreqDelta(to_fit.index)
@@ -202,7 +202,7 @@ def fitPolynomial(
data[field] = residues
if eval_flags:
# TODO: we does not get any flags here, because of masking=field
- worst = flagger[field].rolling(winsz, center=True, min_periods=min_periods).max()
- flagger[field] = worst
+ worst = flags[field].rolling(winsz, center=True, min_periods=min_periods).max()
+ flags[field] = worst
- return data, flagger
+ return data, flags
diff --git a/saqc/funcs/drift.py b/saqc/funcs/drift.py
index 0892673a8..65f5b043d 100644
--- a/saqc/funcs/drift.py
+++ b/saqc/funcs/drift.py
@@ -15,7 +15,7 @@ from scipy.spatial.distance import pdist
from saqc.constants import *
from saqc.core.register import register
-from saqc.core import Flags as Flagger
+from saqc.core import Flags
from saqc.funcs.resampling import shift
from saqc.funcs.changepoints import assignChangePointCluster
from saqc.funcs.tools import drop, copy
@@ -30,7 +30,7 @@ LinkageString = Literal["single", "complete", "average", "weighted", "centroid",
def flagDriftFromNorm(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
fields: Sequence[ColumnName],
segment_freq: FreqString,
norm_spread: float,
@@ -39,7 +39,7 @@ def flagDriftFromNorm(
linkage_method: LinkageString = "single",
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
The function flags value courses that significantly deviate from a group of normal value courses.
@@ -138,23 +138,23 @@ def flagDriftFromNorm(
drifters = detectDeviants(segment[1], metric, norm_spread, norm_frac, linkage_method, 'variables')
for var in drifters:
- flagger[segment[1].index, fields[var]] = flag
+ flags[segment[1].index, fields[var]] = flag
- return data, flagger
+ return data, flags
@register(masking='all', module="drift")
def flagDriftFromReference(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
fields: Sequence[ColumnName],
segment_freq: FreqString,
thresh: float,
metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
The function flags value courses that deviate from a reference course by a margin exceeding a certain threshold.
@@ -216,16 +216,16 @@ def flagDriftFromReference(
dist = metric(segment[1].iloc[:, i].values, segment[1].loc[:, field].values)
if dist > thresh:
- flagger[segment[1].index, fields[i]] = flag
+ flags[segment[1].index, fields[i]] = flag
- return data, flagger
+ return data, flags
@register(masking='all', module="drift")
def flagDriftFromScaledNorm(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
fields_scale1: Sequence[ColumnName],
fields_scale2: Sequence[ColumnName],
segment_freq: FreqString,
@@ -235,7 +235,7 @@ def flagDriftFromScaledNorm(
linkage_method: LinkageString = "single",
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
The function linearly rescales one set of variables to another set of variables with a different scale and then
flags value courses that significantly deviate from a group of normal value courses.
@@ -334,22 +334,22 @@ def flagDriftFromScaledNorm(
drifters = detectDeviants(segment[1], metric, norm_spread, norm_frac, linkage_method, 'variables')
for var in drifters:
- flagger[segment[1].index, fields[var]] = flag
+ flags[segment[1].index, fields[var]] = flag
- return data, flagger
+ return data, flags
@register(masking='all', module="drift")
def correctExponentialDrift(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
maint_data_field: ColumnName,
cal_mean: int = 5,
flag_maint_period: bool = False,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
The function fits an exponential model to chunks of data[field].
It is assumed, that between maintenance events, there is a drift effect shifting the meassurements in a way, that
@@ -412,7 +412,7 @@ def correctExponentialDrift(
"""
# 1: extract fit intervals:
if data[maint_data_field].empty:
- return data, flagger
+ return data, flags
data = data.copy()
to_correct = data[field]
@@ -446,22 +446,22 @@ def correctExponentialDrift(
to_flag = drift_frame["drift_group"]
to_flag = to_flag.drop(to_flag[: maint_data.index[0]].index)
to_flag = to_flag.dropna()
- flagger[to_flag, field] = flag
+ flags[to_flag, field] = flag
- return data, flagger
+ return data, flags
@register(masking='all', module="drift")
def correctRegimeAnomaly(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
cluster_field: ColumnName,
model: CurveFitter,
regime_transmission: Optional[FreqString] = None,
x_date: bool = False,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Function fits the passed model to the different regimes in data[field] and tries to correct
those values, that have assigned a negative label by data[cluster_field].
@@ -561,21 +561,21 @@ def correctRegimeAnomaly(
last_valid = 1
data[field] = data_ser
- return data, flagger
+ return data, flags
@register(masking='all', module="drift")
def correctOffset(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
max_mean_jump: float,
normal_spread: float,
search_winsz: FreqString,
min_periods: int,
regime_transmission: Optional[FreqString] = None,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Parameters
----------
@@ -609,23 +609,23 @@ def correctOffset(
flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
"""
- data, flagger = copy(data, field, flagger, field + '_CPcluster')
- data, flagger = assignChangePointCluster(
- data, field + '_CPcluster', flagger,
+ data, flags = copy(data, field, flags, field + '_CPcluster')
+ data, flags = assignChangePointCluster(
+ data, field + '_CPcluster', flags,
lambda x, y: np.abs(np.mean(x) - np.mean(y)),
lambda x, y: max_mean_jump,
bwd_window=search_winsz,
min_periods_bwd=min_periods
)
- data, flagger = assignRegimeAnomaly(data, field, flagger, field + '_CPcluster', normal_spread)
- data, flagger = correctRegimeAnomaly(
- data, field, flagger, field + '_CPcluster',
+ data, flags = assignRegimeAnomaly(data, field, flags, field + '_CPcluster', normal_spread)
+ data, flags = correctRegimeAnomaly(
+ data, field, flags, field + '_CPcluster',
lambda x, p1: np.array([p1] * x.shape[0]),
regime_transmission=regime_transmission
)
- data, flagger = drop(data, field + '_CPcluster', flagger)
+ data, flags = drop(data, field + '_CPcluster', flags)
- return data, flagger
+ return data, flags
def _driftFit(x, shift_target, cal_mean):
@@ -660,7 +660,7 @@ def _driftFit(x, shift_target, cal_mean):
def flagRegimeAnomaly(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
cluster_field: ColumnName,
norm_spread: float,
linkage_method: LinkageString = "single",
@@ -668,7 +668,7 @@ def flagRegimeAnomaly(
norm_frac: float = 0.5,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
A function to flag values belonging to an anomalous regime regarding modelling regimes of field.
@@ -716,7 +716,7 @@ def flagRegimeAnomaly(
Flags values may have changed, relatively to the flagger input.
"""
return assignRegimeAnomaly(
- data, field, flagger,
+ data, field, flags,
cluster_field,
norm_spread,
linkage_method=linkage_method,
@@ -733,7 +733,7 @@ def flagRegimeAnomaly(
def assignRegimeAnomaly(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
cluster_field: ColumnName,
norm_spread: float,
linkage_method: LinkageString = "single",
@@ -743,7 +743,7 @@ def assignRegimeAnomaly(
set_flags: bool = False,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
A function to detect values belonging to an anomalous regime regarding modelling regimes of field.
@@ -805,7 +805,7 @@ def assignRegimeAnomaly(
if set_flags:
for p in plateaus:
- flagger[cluster_dios.iloc[:, p].index, field] = flag
+ flags[cluster_dios.iloc[:, p].index, field] = flag
if set_cluster:
for p in plateaus:
@@ -813,4 +813,4 @@ def assignRegimeAnomaly(
series[series == cluster[p]] = -cluster[p]
data[cluster_field] = series
- return data, flagger
+ return data, flags
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index db0ce930d..94b04da10 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -7,14 +7,14 @@ from dios import DictOfSeries
from saqc.constants import *
from saqc.lib.types import *
-from saqc.core import register, Flags as Flagger
+from saqc.core import register, Flags
import warnings
@register(masking='field', module="flagtools")
def forceFlags(
- data: DictOfSeries, field: ColumnName, flagger: Flagger, flag: float = BAD, **kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+ data: DictOfSeries, field: ColumnName, flags: Flags, flag: float = BAD, **kwargs
+) -> Tuple[DictOfSeries, Flags]:
"""
Set whole column to a flag value.
@@ -41,13 +41,13 @@ def forceFlags(
clearFlags : set whole column to UNFLAGGED
flagUnflagged : set flag value at all unflagged positions
"""
- flagger[:, field] = flag
- return data, flagger
+ flags[:, field] = flag
+ return data, flags
# masking='none' is sufficient because call is redirected
@register(masking='none', module="flagtools")
-def clearFlags(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+def clearFlags(data: DictOfSeries, field: ColumnName, flags: Flags, **kwargs) -> Tuple[DictOfSeries, Flags]:
"""
Set whole column to UNFLAGGED.
@@ -77,13 +77,13 @@ def clearFlags(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs
flag = kwargs.pop('flag')
warnings.warn(f'`flag={flag}` is ignored here.')
- return forceFlags(data, field, flagger, flag=UNFLAGGED, **kwargs)
+ return forceFlags(data, field, flags, flag=UNFLAGGED, **kwargs)
@register(masking='field', module="flagtools")
def flagUnflagged(
- data: DictOfSeries, field: ColumnName, flagger: Flagger, flag: float = BAD, **kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+ data: DictOfSeries, field: ColumnName, flags: Flags, flag: float = BAD, **kwargs
+) -> Tuple[DictOfSeries, Flags]:
"""
Function sets a flag at all unflagged positions.
@@ -112,13 +112,13 @@ def flagUnflagged(
clearFlags : set whole column to UNFLAGGED
forceFlags : set whole column to a flag value
"""
- unflagged = flagger[field].isna() | (flagger[field] == UNFLAGGED)
- flagger[unflagged, field] = flag
- return data, flagger
+ unflagged = flags[field].isna() | (flags[field] == UNFLAGGED)
+ flags[unflagged, field] = flag
+ return data, flags
@register(masking='field', module="flagtools")
-def flagGood(data: DictOfSeries, field: ColumnName, flagger: Flagger, flag=BAD, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+def flagGood(data: DictOfSeries, field: ColumnName, flags: Flags, flag=BAD, **kwargs) -> Tuple[DictOfSeries, Flags]:
"""
Function sets the GOOD flag at all unflagged positions.
@@ -139,18 +139,18 @@ def flagGood(data: DictOfSeries, field: ColumnName, flagger: Flagger, flag=BAD,
The flagger object, holding flags and additional Informations related to `data`.
"""
warnings.warn("'flagGood' is deprecated and does nothing, use 'flagUnflagged' instead", DeprecationWarning)
- return data, flagger
+ return data, flags
@register(masking='field', module="flagtools")
def flagManual(
- data: DictOfSeries, field: ColumnName, flagger: Flagger,
+ data: DictOfSeries, field: ColumnName, flags: Flags,
mdata: Union[pd.Series, pd.DataFrame, DictOfSeries],
mflag: Any = 1,
method: Literal["plain", "ontime", "left-open", "right-open"] = 'plain',
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Flag data by given, "manually generated" data.
@@ -280,12 +280,12 @@ def flagManual(
mask = mdata == mflag
mask = mask.reindex(dat.index).fillna(False)
- flagger[mask, field] = flag
- return data, flagger
+ flags[mask, field] = flag
+ return data, flags
@register(masking='none', module="flagtools")
-def flagDummy(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+def flagDummy(data: DictOfSeries, field: ColumnName, flags: Flags, **kwargs) -> Tuple[DictOfSeries, Flags]:
"""
Function does nothing but returning data and flagger.
@@ -305,11 +305,11 @@ def flagDummy(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs
flagger : saqc.flagger.Flagger
The flagger object, holding flags and additional Informations related to `data`.
"""
- return data, flagger
+ return data, flags
@register(masking='none', module="flagtools")
-def flagForceFail(data: DictOfSeries, field: ColumnName, flagger: Flagger, **kwargs):
+def flagForceFail(data: DictOfSeries, field: ColumnName, flags: Flags, **kwargs):
"""
Function raises a runtime error.
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index 095ae1eea..329514fcf 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -11,7 +11,7 @@ import pandas as pd
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core import register, initFlagsLike, Flags as Flagger
+from saqc.core import register, initFlagsLike, Flags
from saqc.core.visitor import ENVIRONMENT
import operator as op
@@ -20,7 +20,7 @@ _OP = {'<': op.lt, '<=': op.le, '==': op.eq, '!=': op.ne, '>': op.gt, '>=': op.g
def _dslIsFlagged(
- flagger: Flagger, var: pd.Series, flag: float = None, comparator: str = None
+ flags: Flags, var: pd.Series, flag: float = None, comparator: str = None
) -> Union[pd.Series, DictOfSeries]:
"""
helper function for `flag`
@@ -46,10 +46,10 @@ def _dslIsFlagged(
comparator = '>='
_op = _OP[comparator]
- return _op(flagger[var.name], flag)
+ return _op(flags[var.name], flag)
-def _execGeneric(flagger: Flagger, data: DictOfSeries, func: Callable[[pd.Series], pd.Series], field: str,
+def _execGeneric(flags: Flags, data: DictOfSeries, func: Callable[[pd.Series], pd.Series], field: str,
nodata: float) -> pd.Series:
# TODO:
# - check series.index compatibility
@@ -65,7 +65,7 @@ def _execGeneric(flagger: Flagger, data: DictOfSeries, func: Callable[[pd.Series
args.append(data[k])
globs = {
- "isflagged": partial(_dslIsFlagged, flagger),
+ "isflagged": partial(_dslIsFlagged, flags),
"ismissing": lambda var: ((var == nodata) | pd.isnull(var)),
"mask": lambda cond: data[cond.name].mask(cond),
"this": field,
@@ -83,11 +83,11 @@ def _execGeneric(flagger: Flagger, data: DictOfSeries, func: Callable[[pd.Series
def process(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
func: Callable[[pd.Series], pd.Series],
nodata: float = np.nan,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
generate/process data with generically defined functions.
@@ -137,27 +137,27 @@ def process(
>>> lambda temperature, uncertainty: np.round(temperature) * np.sqrt(uncertainty)
"""
- data[field] = _execGeneric(flagger, data, func, field, nodata).squeeze()
+ data[field] = _execGeneric(flags, data, func, field, nodata).squeeze()
# TODO: the former comment wished to overwrite the column, but i'm not sure -- palmb
# see #GL177
- if field in flagger:
- flagger.drop(field)
+ if field in flags:
+ flags.drop(field)
- flagger[field] = initFlagsLike(data[field])[field]
- return data, flagger
+ flags[field] = initFlagsLike(data[field])[field]
+ return data, flags
@register(masking='all', module="generic")
def flag(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
func: Callable[[pd.Series], pd.Series],
nodata: float = np.nan,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
# TODO : fix docstring, check if all still works
"""
a function to flag a data column by evaluation of a generic expression.
@@ -238,18 +238,18 @@ def flag(
# NOTE:
# The naming of the func parameter is pretty confusing
# as it actually holds the result of a generic expression
- mask = _execGeneric(flagger, data, func, field, nodata).squeeze()
+ mask = _execGeneric(flags, data, func, field, nodata).squeeze()
if np.isscalar(mask):
raise TypeError(f"generic expression does not return an array")
if not np.issubdtype(mask.dtype, np.bool_):
raise TypeError(f"generic expression does not return a boolean array")
- if field not in flagger.columns:
- flagger[field] = pd.Series(UNFLAGGED, index=mask.index, name=field)
+ if field not in flags.columns:
+ flags[field] = pd.Series(UNFLAGGED, index=mask.index, name=field)
# if flagger.getFlags(field).empty:
# flagger = flagger.merge(
# flagger.initFlags(
# data=pd.Series(name=field, index=mask.index, dtype=np.float64)))
- flagger[mask, field] = flag
- return data, flagger
+ flags[mask, field] = flag
+ return data, flags
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index 9f3b985d7..32fcd2ba4 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -7,7 +7,7 @@ import pandas as pd
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core import register, Flags as Flagger
+from saqc.core import register, Flags
from saqc.core.register import _isflagged
from saqc.core.history import applyFunctionOnHistory
from saqc.lib.ts_operators import interpolateNANs
@@ -20,14 +20,14 @@ _SUPPORTED_METHODS = Literal[
@register(masking='field', module="interpolation")
def interpolateByRolling(
- data: DictOfSeries, field: str, flagger: Flagger,
+ data: DictOfSeries, field: str, flags: Flags,
winsz: Union[str, int],
func: Callable[[pd.Series], float] = np.median,
center: bool = True,
min_periods: int = 0,
flag: float = UNFLAGGED,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Interpolates nan-values in the data by assigning them the aggregation result of the window surrounding them.
@@ -87,23 +87,23 @@ def interpolateByRolling(
data[field] = datcol
if flag is not None:
- flagger[interpolated, field] = flag
+ flags[interpolated, field] = flag
- return data, flagger
+ return data, flags
@register(masking='field', module="interpolation")
def interpolateInvalid(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
method: _SUPPORTED_METHODS,
inter_order: int = 2,
inter_limit: int = 2,
downgrade_interpolation: bool = False,
flag: float = UNFLAGGED,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Function to interpolate nan values in the data.
@@ -160,10 +160,10 @@ def interpolateInvalid(
interpolated = data[field].isna() & inter_data.notna()
if flag is not None:
- flagger[interpolated, field] = flag
+ flags[interpolated, field] = flag
data[field] = inter_data
- return data, flagger
+ return data, flags
def _resampleOverlapping(data: pd.Series, freq: str, fill_value):
@@ -181,14 +181,14 @@ def _resampleOverlapping(data: pd.Series, freq: str, fill_value):
def interpolateIndex(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
freq: str,
method: _SUPPORTED_METHODS,
inter_order: int = 2,
inter_limit: int = 2,
downgrade_interpolation: bool = False,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Function to interpolate the data at regular (equidistant) timestamps (or Grid points).
@@ -238,14 +238,14 @@ def interpolateIndex(
Flags values and shape may have changed relatively to the flagger input.
"""
if data[field].empty:
- return data, flagger
+ return data, flags
datcol = data[field].copy()
start, end = datcol.index[0].floor(freq), datcol.index[-1].ceil(freq)
grid_index = pd.date_range(start=start, end=end, freq=freq, name=datcol.index.name)
- flagged = _isflagged(flagger[field], kwargs['to_mask'])
+ flagged = _isflagged(flags[field], kwargs['to_mask'])
# drop all points that hold no relevant grid information
datcol = datcol[~flagged].dropna()
@@ -275,11 +275,11 @@ def interpolateIndex(
data[field] = inter_data[grid_index]
# do the reshaping on the history
- flagger.history[field] = applyFunctionOnHistory(
- flagger.history[field],
+ flags.history[field] = applyFunctionOnHistory(
+ flags.history[field],
hist_func=_resampleOverlapping, hist_kws=dict(freq=freq, fill_value=UNFLAGGED),
mask_func=_resampleOverlapping, mask_kws=dict(freq=freq, fill_value=False),
last_column='dummy'
)
- return data, flagger
+ return data, flags
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index e6f804be8..844643f61 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -12,7 +12,7 @@ from outliers import smirnov_grubbs
from scipy.optimize import curve_fit
from saqc.constants import *
-from saqc.core import register, Flags as Flagger
+from saqc.core import register, Flags
from saqc.lib.types import ColumnName, FreqString, IntegerWindow
from saqc.lib.tools import customRoller, findIndex, getFreqDelta
from saqc.funcs.scores import assignKNNScore
@@ -23,14 +23,14 @@ import saqc.lib.ts_operators as ts_ops
def flagByStray(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
partition_freq: Optional[Union[IntegerWindow, FreqString]] = None,
partition_min: int = 11,
iter_start: float = 0.5,
alpha: float = 0.05,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Flag outliers in 1-dimensional (score) data with the STRAY Algorithm.
@@ -79,7 +79,7 @@ def flagByStray(
scores = data[field].dropna()
if scores.empty:
- return data, flagger
+ return data, flags
if not partition_freq:
partition_freq = scores.shape[0]
@@ -117,16 +117,16 @@ def flagByStray(
for iter_index in range(i_start - 1, sample_size):
if gaps[iter_index] > log_alpha * ghat[iter_index]:
index = partition.index[sorted_i[iter_index:]]
- flagger[index, field] = flag
+ flags[index, field] = flag
break
- return data, flagger
+ return data, flags
def _evalStrayLabels(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
fields: Sequence[str],
reduction_range: Optional[str] = None,
reduction_drop_flagged: bool = False, # TODO: still a case ?
@@ -135,7 +135,7 @@ def _evalStrayLabels(
at_least_one: bool = True,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
The function "reduces" an observations flag to components of it, by applying MAD (See references)
test onto every components temporal surrounding.
@@ -178,14 +178,14 @@ def _evalStrayLabels(
[1] https://www.itl.nist.gov/div898/handbook/eda/section3/eda35h.htm
"""
val_frame = data[fields].to_df()
- stray_detects = flagger[field] > UNFLAGGED
+ stray_detects = flags[field] > UNFLAGGED
stray_detects = stray_detects[stray_detects]
to_flag_frame = pd.DataFrame(False, columns=fields, index=stray_detects.index)
if reduction_range is None:
for field in to_flag_frame.columns:
- flagger[to_flag_frame.index, field] = flag
- return data, flagger
+ flags[to_flag_frame.index, field] = flag
+ return data, flags
for var in fields:
for index in enumerate(to_flag_frame.index):
@@ -232,9 +232,9 @@ def _evalStrayLabels(
for field in to_flag_frame.columns:
col = to_flag_frame[field]
- flagger[col[col].index, field] = flag
+ flags[col[col].index, field] = flag
- return data, flagger
+ return data, flags
def _expFit(val_frame, scoring_method="kNNMaxGap", n_neighbors=10, iter_start=0.5, alpha=0.05, bin_frac=10):
@@ -352,7 +352,7 @@ def _expFit(val_frame, scoring_method="kNNMaxGap", n_neighbors=10, iter_start=0.
def flagMVScores(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
fields: Sequence[ColumnName],
trafo: Callable[[pd.Series], pd.Series] = lambda x: x,
alpha: float = 0.05,
@@ -368,7 +368,7 @@ def flagMVScores(
reduction_min_periods: int = 1,
flag: float = BAD,
**kwargs,
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
The algorithm implements a 3-step outlier detection procedure for simultaneously flagging of higher dimensional
data (dimensions > 3).
@@ -473,8 +473,8 @@ def flagMVScores(
outliers. See description of the `threshing` parameter for more details. Although [2] gives a fully detailed
overview over the `stray` algorithm.
"""
- data, flagger = assignKNNScore(
- data, 'dummy', flagger,
+ data, flags = assignKNNScore(
+ data, 'dummy', flags,
fields=fields,
n_neighbors=n_neighbors,
trafo=trafo,
@@ -485,8 +485,8 @@ def flagMVScores(
kNN_algorithm='ball_tree',
partition_min=stray_partition_min, **kwargs)
- data, flagger = flagByStray(
- data, 'kNN_scores', flagger,
+ data, flags = flagByStray(
+ data, 'kNN_scores', flags,
partition_freq=stray_partition,
partition_min=stray_partition_min,
iter_start=iter_start,
@@ -494,8 +494,8 @@ def flagMVScores(
flag=flag,
**kwargs)
- data, flagger = _evalStrayLabels(
- data, 'kNN_scores', flagger,
+ data, flags = _evalStrayLabels(
+ data, 'kNN_scores', flags,
fields=fields,
reduction_range=reduction_range,
reduction_drop_flagged=reduction_drop_flagged,
@@ -504,14 +504,14 @@ def flagMVScores(
flag=flag,
**kwargs)
- return data, flagger
+ return data, flags
@register(masking='field', module="outliers")
def flagRaise(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
thresh: float,
raise_window: FreqString,
intended_freq: FreqString,
@@ -522,7 +522,7 @@ def flagRaise(
numba_boost: bool = True, # TODO: rm, not a user decision
flag: float = BAD,
**kwargs,
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
The function flags raises and drops in value courses, that exceed a certain threshold
within a certain timespan.
@@ -629,7 +629,7 @@ def flagRaise(
raise_series = raise_series.apply(raise_check, args=(thresh,), raw=True)
if raise_series.isna().all():
- return data, flagger
+ return data, flags
# "unflag" values of insufficient deviation to their predecessors
if min_slope is not None:
@@ -672,21 +672,21 @@ def flagRaise(
# check means against critical raise value:
to_flag = dataseries >= weighted_rolling_mean + (raise_series / mean_raise_factor)
to_flag &= raise_series.notna()
- flagger[to_flag[to_flag].index, field] = flag
+ flags[to_flag[to_flag].index, field] = flag
- return data, flagger
+ return data, flags
@register(masking='field', module="outliers")
def flagMAD(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
window: FreqString,
z: float = 3.5,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
The function represents an implementation of the modyfied Z-score outlier detection method.
@@ -739,15 +739,15 @@ def flagMAD(
index = mask.index
mask.loc[index < index[0] + pd.to_timedelta(window)] = False
- flagger[mask, field] = flag
- return data, flagger
+ flags[mask, field] = flag
+ return data, flags
@register(masking='field', module="outliers")
def flagOffset(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
thresh: float,
tolerance: float,
window: Union[IntegerWindow, FreqString],
@@ -755,7 +755,7 @@ def flagOffset(
numba_kickin: int = 200000, # TODO: rm, not a user decision
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
A basic outlier test that is designed to work for harmonized and not harmonized data.
@@ -842,7 +842,7 @@ def flagOffset(
post_jumps = post_jumps[post_jumps]
if post_jumps.empty:
- return data, flagger
+ return data, flags
# get all the entries preceding a significant jump and its successors within "length" range
to_roll = post_jumps.reindex(dataseries.index, method="bfill", tolerance=window, fill_value=False).dropna()
@@ -897,22 +897,22 @@ def flagOffset(
cresult = calcResult(result)
cresult = cresult[cresult].index
- flagger[cresult, field] = flag
- return data, flagger
+ flags[cresult, field] = flag
+ return data, flags
@register(masking='field', module="outliers")
def flagByGrubbs(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
winsz: Union[FreqString, IntegerWindow],
alpha: float = 0.05,
min_periods: int = 8,
check_lagged: bool = False,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
The function flags values that are regarded outliers due to the grubbs test.
@@ -1006,20 +1006,20 @@ def flagByGrubbs(
to_flag &= to_flag_lagged
- flagger[to_flag, field] = flag
- return data, flagger
+ flags[to_flag, field] = flag
+ return data, flags
@register(masking='field', module="outliers")
def flagRange(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
min: float = -np.inf,
max: float = np.inf,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Function flags values not covered by the closed interval [`min`, `max`].
@@ -1050,21 +1050,21 @@ def flagRange(
# using .values is much faster
datacol = data[field].values
mask = (datacol < min) | (datacol > max)
- flagger[mask, field] = flag
- return data, flagger
+ flags[mask, field] = flag
+ return data, flags
@register(masking='all', module="outliers")
def flagCrossStatistic(
data: DictOfSeries,
field: ColumnName,
- flagger: Flagger,
+ flags: Flags,
fields: Sequence[ColumnName],
thresh: float,
cross_stat: Literal["modZscore", "Zscore"] = "modZscore",
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Function checks for outliers relatively to the "horizontal" input data axis.
@@ -1139,6 +1139,6 @@ def flagCrossStatistic(
mask = diff_scores > thresh
for var in fields:
- flagger[mask[var], var] = flag
+ flags[mask[var], var] = flag
- return data, flagger
+ return data, flags
diff --git a/saqc/funcs/pattern.py b/saqc/funcs/pattern.py
index a51a157a4..5f4829e9c 100644
--- a/saqc/funcs/pattern.py
+++ b/saqc/funcs/pattern.py
@@ -9,7 +9,7 @@ from mlxtend.evaluate import permutation_test
from dios.dios import DictOfSeries
from saqc.constants import *
-from saqc.core import register, Flags as Flagger
+from saqc.core import register, Flags
from saqc.lib.tools import customRoller
@@ -17,13 +17,13 @@ from saqc.lib.tools import customRoller
def flagPatternByDTW(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
ref_field: str,
widths: Sequence[int] = (1, 2, 4, 8),
waveform: str = "mexh",
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Pattern recognition via wavelets.
@@ -97,21 +97,21 @@ def flagPatternByDTW(
sz = len(ref)
mask = customRoller(dat, window=sz, min_periods=sz).apply(isPattern, raw=True)
- flagger[mask, field] = flag
- return data, flagger
+ flags[mask, field] = flag
+ return data, flags
@register(masking='field', module="pattern")
def flagPatternByWavelet(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
ref_field: str,
max_distance: float = 0.03,
normalize: bool = True,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
""" Pattern Recognition via Dynamic Time Warping.
The steps are:
@@ -169,5 +169,5 @@ def flagPatternByWavelet(
sz = len(ref)
mask = customRoller(dat, window=sz, min_periods=sz).apply(isPattern, raw=True)
- flagger[mask, field] = flag
- return data, flagger
+ flags[mask, field] = flag
+ return data, flags
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 466bcf382..967966c9c 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -9,7 +9,7 @@ import pandas as pd
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core import register, Flags as Flagger
+from saqc.core import register, Flags
from saqc.core.register import _isflagged
from saqc.core.history import applyFunctionOnHistory
from saqc.lib.tools import evalFreqStr, getFreqDelta
@@ -35,14 +35,14 @@ METHOD2ARGS = {
def aggregate(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
freq: str,
value_func,
flag_func: Callable[[pd.Series], float] = np.nanmax,
method: Literal["fagg", "bagg", "nagg"] = "nagg",
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
A method to "regularize" data by aggregating (resampling) data at a regular timestamp.
@@ -106,9 +106,9 @@ def aggregate(
Flags values and shape may have changed relatively to the flagger input.
"""
- data, flagger = copy(data, field, flagger, field + '_original')
+ data, flags = copy(data, field, flags, field + '_original')
return resample(
- data, field, flagger,
+ data, field, flags,
freq=freq,
agg_func=value_func,
flag_agg_func=flag_func,
@@ -122,10 +122,10 @@ def aggregate(
def linear(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
freq: str,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
A method to "regularize" data by interpolating linearly the data at regular timestamp.
@@ -165,20 +165,20 @@ def linear(
Flags values and shape may have changed relatively to the flagger input.
"""
- data, flagger = copy(data, field, flagger, field + '_original')
- return interpolateIndex(data, field, flagger, freq, "time", **kwargs)
+ data, flags = copy(data, field, flags, field + '_original')
+ return interpolateIndex(data, field, flags, freq, "time", **kwargs)
@register(masking='none', module="resampling")
def interpolate(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
freq: str,
method: _SUPPORTED_METHODS,
order: int = 1,
**kwargs,
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
A method to "regularize" data by interpolating the data at regular timestamp.
@@ -232,22 +232,22 @@ def interpolate(
Flags values and shape may have changed relatively to the flagger input.
"""
- data, flagger = copy(data, field, flagger, field + '_original')
- return interpolateIndex(data, field, flagger, freq, method=method, inter_order=order, **kwargs)
+ data, flags = copy(data, field, flags, field + '_original')
+ return interpolateIndex(data, field, flags, freq, method=method, inter_order=order, **kwargs)
@register(masking='none', module="resampling")
def mapToOriginal(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
method: Literal[
"inverse_fagg", "inverse_bagg", "inverse_nagg",
"inverse_fshift", "inverse_bshift", "inverse_nshift",
"inverse_interpolation"
],
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
The Function function "undoes" regularization, by regaining the original data and projecting the
flags calculated for the regularized data onto the original ones.
@@ -312,21 +312,21 @@ def mapToOriginal(
Flags values and shape may have changed relatively to the flagger input.
"""
newfield = str(field) + '_original'
- data, flagger = reindexFlags(data, newfield, flagger, method, source=field, to_mask=False)
- data, flagger = drop(data, field, flagger)
- return rename(data, newfield, flagger, field)
+ data, flags = reindexFlags(data, newfield, flags, method, source=field, to_mask=False)
+ data, flags = drop(data, field, flags)
+ return rename(data, newfield, flags, field)
@register(masking='none', module="resampling")
def shift(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
freq: str,
method: Literal["fshift", "bshift", "nshift"] = "nshift",
freq_check: Optional[Literal["check", "auto"]] = None, # TODO: not a user decision
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Function to shift data and flags to a regular (equidistant) timestamp grid, according to ``method``.
@@ -370,19 +370,19 @@ def shift(
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
- data, flagger = copy(data, field, flagger, field + '_original')
- return _shift(data, field, flagger, freq, method=method, freq_check=freq_check, **kwargs)
+ data, flags = copy(data, field, flags, field + '_original')
+ return _shift(data, field, flags, freq, method=method, freq_check=freq_check, **kwargs)
def _shift(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
freq: str,
method: Literal["fshift", "bshift", "nshift"] = "nshift",
freq_check: Optional[Literal["check", "auto"]] = None,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Function to shift data points to regular (equidistant) timestamps.
@@ -390,7 +390,7 @@ def _shift(
--------
shift : Main caller, docstring
"""
- flagged = _isflagged(flagger[field], kwargs['to_mask'])
+ flagged = _isflagged(flags[field], kwargs['to_mask'])
datcol = data[field]
datcol[flagged] = np.nan
freq = evalFreqStr(freq, freq_check, datcol.index)
@@ -399,7 +399,7 @@ def _shift(
datcol = shift2Freq(datcol, method, freq, fill_value=np.nan)
# do the shift on the history
- history = flagger.history[field]
+ history = flags.history[field]
history.hist = shift2Freq(history.hist, method, freq, fill_value=UNTOUCHED)
history.mask = shift2Freq(history.mask, method, freq, fill_value=False)
@@ -409,16 +409,16 @@ def _shift(
dummy = pd.Series(UNTOUCHED, index=datcol.index, dtype=float)
history.append(dummy, force=True)
- flagger.history[field] = history
+ flags.history[field] = history
data[field] = datcol
- return data, flagger
+ return data, flags
@register(masking='none', module="resampling")
def resample(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
freq: str,
agg_func: Callable[[pd.Series], pd.Series] = np.mean,
method: Literal["fagg", "bagg", "nagg"] = "bagg",
@@ -429,7 +429,7 @@ def resample(
flag_agg_func: Callable[[pd.Series], float] = max,
freq_check: Optional[Literal["check", "auto"]] = None,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Function to resample the data. Afterwards the data will be sampled at regular (equidistant) timestamps
(or Grid points). Sampling intervals therefor get aggregated with a function, specifyed by 'agg_func' parameter and
@@ -513,7 +513,7 @@ def resample(
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
- flagged = _isflagged(flagger[field], kwargs['to_mask'])
+ flagged = _isflagged(flags[field], kwargs['to_mask'])
datcol = data[field]
datcol[flagged] = np.nan
freq = evalFreqStr(freq, freq_check, datcol.index)
@@ -537,15 +537,15 @@ def resample(
max_invalid_consec=max_invalid_consec_f,
)
- flagger.history[field] = applyFunctionOnHistory(
- flagger.history[field],
+ flags.history[field] = applyFunctionOnHistory(
+ flags.history[field],
hist_func=aggregate2Freq, hist_kws=kws,
mask_func=aggregate2Freq, mask_kws=kws,
last_column='dummy'
)
data[field] = datcol
- return data, flagger
+ return data, flags
def _getChunkBounds(target: pd.Series, flagscol: pd.Series, freq: str):
@@ -602,7 +602,7 @@ def _inverseShift(source: pd.Series, target: pd.Series, drop_mask: pd.Series,
def reindexFlags(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
method: Literal[
"inverse_fagg", "inverse_bagg", "inverse_nagg",
"inverse_fshift", "inverse_bshift", "inverse_nshift"
@@ -610,7 +610,7 @@ def reindexFlags(
source: str,
freq: Optional[str] = None,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
The Function projects flags of "source" onto flags of "field". Wherever the "field" flags are "better" then the
source flags projected on them, they get overridden with this associated source flag value.
@@ -672,7 +672,7 @@ def reindexFlags(
The flagger object, holding flags and additional Informations related to `data`.
Flags values and shape may have changed relatively to the flagger input.
"""
- flagscol = flagger[source]
+ flagscol = flags[source]
if freq is None:
freq = getFreqDelta(flagscol.index)
@@ -681,7 +681,7 @@ def reindexFlags(
'projection range to freq parameter')
target_datcol = data[field]
- target_flagscol = flagger[field]
+ target_flagscol = flags[field]
dummy = pd.Series(np.nan, target_flagscol.index, dtype=float)
if method[-13:] == "interpolation":
@@ -709,6 +709,6 @@ def reindexFlags(
else:
raise ValueError(f"unknown method {method}")
- history = applyFunctionOnHistory(flagger.history[source], func, func_kws, func, mask_kws, last_column=dummy)
- flagger.history[field] = flagger.history[field].append(history, force=False)
- return data, flagger
+ history = applyFunctionOnHistory(flags.history[source], func, func_kws, func, mask_kws, last_column=dummy)
+ flags.history[field] = flags.history[field].append(history, force=False)
+ return data, flags
diff --git a/saqc/funcs/residues.py b/saqc/funcs/residues.py
index b58c0cdf3..ad7b88a64 100644
--- a/saqc/funcs/residues.py
+++ b/saqc/funcs/residues.py
@@ -7,7 +7,7 @@ import numpy as np
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core import register, Flags as Flagger
+from saqc.core import register, Flags
from saqc.funcs.rolling import roll
from saqc.funcs.curvefit import fitPolynomial
@@ -16,7 +16,7 @@ from saqc.funcs.curvefit import fitPolynomial
def calculatePolynomialResidues(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
winsz: Union[str, int],
polydeg: int,
numba: Literal[True, False, "auto"] = "auto", # TODO: rm, not a a user decision
@@ -24,7 +24,7 @@ def calculatePolynomialResidues(
min_periods: Optional[int] = 0,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Function fits a polynomial model to the data and returns the residues.
@@ -101,7 +101,7 @@ def calculatePolynomialResidues(
"""
return fitPolynomial(
- data, field, flagger,
+ data, field, flags,
winsz=winsz,
polydeg=polydeg,
numba=numba,
@@ -117,7 +117,7 @@ def calculatePolynomialResidues(
def calculateRollingResidues(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
winsz: Union[str, int],
func: Callable[[np.ndarray], np.ndarray] = np.mean,
eval_flags: bool = True,
@@ -125,10 +125,10 @@ def calculateRollingResidues(
center: bool = True,
flag: float = BAD,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
""" TODO: docstring needed"""
return roll(
- data, field, flagger,
+ data, field, flags,
winsz=winsz,
func=func,
eval_flags=eval_flags,
diff --git a/saqc/funcs/rolling.py b/saqc/funcs/rolling.py
index 6990bb72f..4b8a5f64e 100644
--- a/saqc/funcs/rolling.py
+++ b/saqc/funcs/rolling.py
@@ -7,7 +7,7 @@ import pandas as pd
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core import register, Flags as Flagger
+from saqc.core import register, Flags
from saqc.lib.tools import getFreqDelta
@@ -15,7 +15,7 @@ from saqc.lib.tools import getFreqDelta
def roll(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
winsz: Union[str, int],
func: Callable[[pd.Series], float]=np.mean,
eval_flags: bool=True, # TODO: not applicable anymore
@@ -73,7 +73,7 @@ def roll(
data = data.copy()
to_fit = data[field]
if to_fit.empty:
- return data, flagger
+ return data, flags
regular = getFreqDelta(to_fit.index)
# starting with the annoying case: finding the rolling interval centers of not-harmonized input time series:
@@ -123,7 +123,7 @@ def roll(
data[field] = means
if eval_flags:
# TODO: we does not get any flags here, because of masking=field
- worst = flagger[field].rolling(winsz, center=True, min_periods=min_periods).max()
- flagger[field] = worst
+ worst = flags[field].rolling(winsz, center=True, min_periods=min_periods).max()
+ flags[field] = worst
- return data, flagger
+ return data, flags
diff --git a/saqc/funcs/scores.py b/saqc/funcs/scores.py
index f1690e0fa..d5b192aa6 100644
--- a/saqc/funcs/scores.py
+++ b/saqc/funcs/scores.py
@@ -7,7 +7,7 @@ import pandas as pd
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core import register, Flags as Flagger
+from saqc.core import register, Flags
from saqc.lib.tools import toSequence
import saqc.lib.ts_operators as ts_ops
@@ -16,7 +16,7 @@ import saqc.lib.ts_operators as ts_ops
def assignKNNScore(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
fields: Sequence[str],
n_neighbors: int = 10,
trafo: Callable[[pd.Series], pd.Series] = lambda x: x,
@@ -29,7 +29,7 @@ def assignKNNScore(
metric: str = 'minkowski',
p: int = 2,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
TODO: docstring need a rework
Score datapoints by an aggregation of the dictances to their k nearest neighbors.
@@ -123,7 +123,7 @@ def assignKNNScore(
val_frame = val_frame.transform(trafo)
if val_frame.empty:
- return data, flagger
+ return data, flags
# partitioning
if not partition_freq:
@@ -155,9 +155,9 @@ def assignKNNScore(
score_ser[partition.index] = resids
# TODO: this unconditionally overwrite a column, may we should fire a warning ? -- palmb
- if target_field in flagger.columns:
- flagger.drop(target_field)
- flagger[target_field] = pd.Series(UNFLAGGED, index=score_ser.index, dtype=float)
+ if target_field in flags.columns:
+ flags.drop(target_field)
+ flags[target_field] = pd.Series(UNFLAGGED, index=score_ser.index, dtype=float)
data[target_field] = score_ser
- return data, flagger
+ return data, flags
diff --git a/saqc/funcs/tools.py b/saqc/funcs/tools.py
index 4ac072016..90db87055 100644
--- a/saqc/funcs/tools.py
+++ b/saqc/funcs/tools.py
@@ -7,12 +7,12 @@ import numpy as np
from dios import DictOfSeries
from saqc.constants import *
-from saqc.core import register, Flags as Flagger
+from saqc.core import register, Flags
from saqc.lib.tools import periodicMask
@register(masking='none', module="tools")
-def copy(data: DictOfSeries, field: str, flagger: Flagger, new_field: str, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+def copy(data: DictOfSeries, field: str, flags: Flags, new_field: str, **kwargs) -> Tuple[DictOfSeries, Flags]:
"""
The function generates a copy of the data "field" and inserts it under the name field + suffix into the existing
data.
@@ -37,17 +37,17 @@ def copy(data: DictOfSeries, field: str, flagger: Flagger, new_field: str, **kwa
The flagger object, holding flags and additional Informations related to `data`.
Flags shape may have changed relatively to the flagger input.
"""
- if new_field in flagger.columns.union(data.columns):
+ if new_field in flags.columns.union(data.columns):
raise ValueError(f"{field}: field already exist")
data[new_field] = data[field].copy()
# implicit copy in history access
- flagger.history[new_field] = flagger.history[field]
- return data, flagger
+ flags.history[new_field] = flags.history[field]
+ return data, flags
@register(masking='none', module="tools")
-def drop(data: DictOfSeries, field: str, flagger: Flagger, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+def drop(data: DictOfSeries, field: str, flags: Flags, **kwargs) -> Tuple[DictOfSeries, Flags]:
"""
The function drops field from the data dios and the flagger.
@@ -70,12 +70,12 @@ def drop(data: DictOfSeries, field: str, flagger: Flagger, **kwargs) -> Tuple[Di
Flags shape may have changed relatively to the flagger input.
"""
del data[field]
- del flagger[field]
- return data, flagger
+ del flags[field]
+ return data, flags
@register(masking='none', module="tools")
-def rename(data: DictOfSeries, field: str, flagger: Flagger, new_name: str, **kwargs) -> Tuple[DictOfSeries, Flagger]:
+def rename(data: DictOfSeries, field: str, flags: Flags, new_name: str, **kwargs) -> Tuple[DictOfSeries, Flags]:
"""
The function renames field to new name (in both, the flagger and the data).
@@ -98,24 +98,24 @@ def rename(data: DictOfSeries, field: str, flagger: Flagger, new_name: str, **kw
The flagger object, holding flags and additional Informations related to `data`.
"""
data[new_name] = data[field]
- flagger.history[new_name] = flagger.history[field]
+ flags.history[new_name] = flags.history[field]
del data[field]
- del flagger[field]
- return data, flagger
+ del flags[field]
+ return data, flags
@register(masking='none', module="tools")
def mask(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
mode: Literal["periodic", "mask_var"],
mask_var: Optional[str]=None,
period_start: Optional[str]=None,
period_end: Optional[str]=None,
include_bounds: bool=True,
**kwargs,
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
This function realizes masking within saqc.
@@ -225,5 +225,5 @@ def mask(
raise ValueError("Keyword passed as masking mode is unknown ({})!".format(mode))
data.aloc[to_mask, field] = np.nan
- flagger[to_mask, field] = UNFLAGGED
- return data, flagger
+ flags[to_mask, field] = UNFLAGGED
+ return data, flags
diff --git a/saqc/funcs/transformation.py b/saqc/funcs/transformation.py
index fbda3ea6f..48a072909 100644
--- a/saqc/funcs/transformation.py
+++ b/saqc/funcs/transformation.py
@@ -6,18 +6,18 @@ import numpy as np
import pandas as pd
from dios import DictOfSeries
-from saqc.core import register, Flags as Flagger
+from saqc.core import register, Flags
@register(masking='field', module="transformation")
def transform(
data: DictOfSeries,
field: str,
- flagger: Flagger,
+ flags: Flags,
func: Callable[[pd.Series], pd.Series],
partition_freq: Optional[Union[float, str]] = None,
**kwargs
-) -> Tuple[DictOfSeries, Flagger]:
+) -> Tuple[DictOfSeries, Flags]:
"""
Function to transform data columns with a transformation that maps series onto series of the same length.
@@ -70,4 +70,4 @@ def transform(
val_ser[partition.index] = func(partition)
data[field] = val_ser
- return data, flagger
+ return data, flags
diff --git a/tests/common.py b/tests/common.py
index eddda827d..1a3f501a7 100644
--- a/tests/common.py
+++ b/tests/common.py
@@ -7,18 +7,18 @@ import pandas as pd
import dios
from saqc.constants import *
-from saqc.core import initFlagsLike, Flags as Flagger
+from saqc.core import initFlagsLike, Flags
TESTNODATA = (np.nan, -9999)
-TESTFLAGGER = (Flagger(),)
+TESTFLAGGER = (Flags(),)
-def flagAll(data, field, flagger, **kwargs):
+def flagAll(data, field, flags, **kwargs):
# NOTE: remember to rename flag -> flag_values
- flagger.copy()
- flagger[:, field] = BAD
- return data, flagger
+ flags.copy()
+ flags[:, field] = BAD
+ return data, flags
def initData(cols=2, start_date="2017-01-01", end_date="2017-12-31", freq=None, rows=None):
@@ -42,7 +42,7 @@ def writeIO(content):
return f
-def checkDataFlaggerInvariants(data, flagger, field, identical=True):
+def checkDataFlagsInvariants(data, flags, field, identical=True):
"""
Check all invariants that must hold at any point for
* field
@@ -68,23 +68,23 @@ def checkDataFlaggerInvariants(data, flagger, field, identical=True):
identical (True, default) of just for equality.
"""
assert isinstance(data, dios.DictOfSeries)
- assert isinstance(flagger, Flagger)
+ assert isinstance(flags, Flags)
# all columns in data are in flagger
- assert data.columns.difference(flagger.columns).empty
+ assert data.columns.difference(flags.columns).empty
# ------------------------------------------------------------------------
# below here, we just check on and with field
# ------------------------------------------------------------------------
assert field in data
- assert field in flagger
+ assert field in flags
- assert flagger[field].dtype == float
+ assert flags[field].dtype == float
# `pd.Index.identical` also check index attributes like `freq`
if identical:
- assert data[field].index.identical(flagger[field].index)
+ assert data[field].index.identical(flags[field].index)
else:
- assert data[field].index.equals(flagger[field].index)
+ assert data[field].index.equals(flags[field].index)
diff --git a/tests/core/test_core.py b/tests/core/test_core.py
index a784cdbac..5370f520d 100644
--- a/tests/core/test_core.py
+++ b/tests/core/test_core.py
@@ -39,7 +39,7 @@ def flags(data, optional):
def test_errorHandling(data):
@register(masking='field')
- def raisingFunc(data, field, flagger, **kwargs):
+ def raisingFunc(data, field, flags, **kwargs):
raise TypeError
var1 = data.columns[0]
@@ -73,11 +73,11 @@ def test_sourceTarget():
var1 = data.columns[0]
target = "new"
- pdata, pflagger = SaQC(data).flagAll(field=var1, target=target).getResult(raw=True)
+ pdata, pflags = SaQC(data).flagAll(field=var1, target=target).getResult(raw=True)
assert (pdata[var1] == pdata[target]).all(axis=None)
- assert all(pflagger[var1] == UNFLAGGED)
- assert all(pflagger[target] > UNFLAGGED)
+ assert all(pflags[var1] == UNFLAGGED)
+ assert all(pflags[target] > UNFLAGGED)
@pytest.mark.parametrize("optional", OPTIONAL)
@@ -85,14 +85,14 @@ def test_dtypes(data, flags):
"""
Test if the categorical dtype is preserved through the core functionality
"""
- flagger = initFlagsLike(data)
- flags_raw = flagger.toDios()
+ flags = initFlagsLike(data)
+ flags_raw = flags.toDios()
var1, var2 = data.columns[:2]
- pdata, pflagger = SaQC(data, flags=flags_raw).flagAll(var1).flagAll(var2).getResult(raw=True)
+ pdata, pflags = SaQC(data, flags=flags_raw).flagAll(var1).flagAll(var2).getResult(raw=True)
- for c in pflagger.columns:
- assert pflagger[c].dtype == flagger[c].dtype
+ for c in pflags.columns:
+ assert pflags[c].dtype == flags[c].dtype
def test_plotting(data):
@@ -104,10 +104,10 @@ def test_plotting(data):
"""
pytest.importorskip("matplotlib", reason="requires matplotlib")
field, *_ = data.columns
- flagger = initFlagsLike(data)
- _, flagger_range = flagRange(data, field, flagger, min=10, max=90, flag=BAD)
- data_new, flagger_range = flagRange(data, field, flagger_range, min=40, max=60, flag=DOUBT)
+ flags = initFlagsLike(data)
+ _, flags_range = flagRange(data, field, flags, min=10, max=90, flag=BAD)
+ data_new, flags_range = flagRange(data, field, flags_range, min=40, max=60, flag=DOUBT)
splot._interactive = False
- splot._plotSingleVariable(data, data_new, flagger, flagger_range, sources=[], targets=[data_new.columns[0]])
- splot._plotMultipleVariables(data, data_new, flagger, flagger_range, targets=data_new.columns)
+ splot._plotSingleVariable(data, data_new, flags, flags_range, sources=[], targets=[data_new.columns[0]])
+ splot._plotMultipleVariables(data, data_new, flags, flags_range, targets=data_new.columns)
splot._interactive = True
diff --git a/tests/core/test_creation.py b/tests/core/test_creation.py
index 295d2adfc..b9b931d29 100644
--- a/tests/core/test_creation.py
+++ b/tests/core/test_creation.py
@@ -6,7 +6,7 @@ import dios
def test_init():
- from saqc import SaQC, Flags as Flagger
+ from saqc import SaQC, Flags
arr = np.array([
[0, 1, 2],
@@ -16,5 +16,5 @@ def test_init():
qc = SaQC(data)
assert isinstance(qc, SaQC)
- assert isinstance(qc._flagger, Flagger)
+ assert isinstance(qc._flags, Flags)
assert isinstance(qc._data, dios.DictOfSeries)
diff --git a/tests/flagger/test_flagger.py b/tests/core/test_flagger.py
similarity index 100%
rename from tests/flagger/test_flagger.py
rename to tests/core/test_flagger.py
diff --git a/tests/flagger/test_flags.py b/tests/core/test_flags.py
similarity index 99%
rename from tests/flagger/test_flags.py
rename to tests/core/test_flags.py
index d0d1585bc..79445b487 100644
--- a/tests/flagger/test_flags.py
+++ b/tests/core/test_flags.py
@@ -7,7 +7,7 @@ import pandas as pd
from saqc.constants import *
from saqc.core.flags import Flags
-from tests.flagger.test_history import (
+from tests.core.test_history import (
History,
is_equal as hist_equal,
)
diff --git a/tests/flagger/test_history.py b/tests/core/test_history.py
similarity index 100%
rename from tests/flagger/test_history.py
rename to tests/core/test_history.py
diff --git a/tests/core/test_reader.py b/tests/core/test_reader.py
index e2d80042b..ded1bdf82 100644
--- a/tests/core/test_reader.py
+++ b/tests/core/test_reader.py
@@ -106,8 +106,8 @@ def test_configChecks(data):
var1, _, var3, *_ = data.columns
@register(masking="none")
- def flagFunc(data, field, flagger, arg, opt_arg=None, **kwargs):
- return data, flagger
+ def flagFunc(data, field, flags, arg, opt_arg=None, **kwargs):
+ return data, flags
header = f"{F.VARNAME};{F.TEST}"
tests = [
@@ -131,8 +131,8 @@ def test_supportedArguments(data):
# TODO: necessary?
@register(masking='field')
- def func(data, field, flagger, kwarg, **kwargs):
- return data, flagger
+ def func(data, field, flags, kwarg, **kwargs):
+ return data, flags
var1 = data.columns[0]
diff --git a/tests/flagger/__init__.py b/tests/flagger/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/tests/funcs/test_constants_detection.py b/tests/funcs/test_constants_detection.py
index d6b7a68f8..a7a7b5b82 100644
--- a/tests/funcs/test_constants_detection.py
+++ b/tests/funcs/test_constants_detection.py
@@ -6,7 +6,7 @@ import numpy as np
from saqc.constants import *
from saqc.funcs.constants import flagConstants, flagByVariance
-from saqc.core import initFlagsLike, Flags as Flagger
+from saqc.core import initFlagsLike, Flags
from tests.common import initData
@@ -21,18 +21,18 @@ def data():
def test_constants_flagBasic(data):
expected = np.arange(5, 22)
field, *_ = data.columns
- flagger = initFlagsLike(data)
- data, flagger_result = flagConstants(data, field, flagger, window="15Min", thresh=0.1, flag=BAD)
- flagscol = flagger_result[field]
+ flags = initFlagsLike(data)
+ data, flags_result = flagConstants(data, field, flags, window="15Min", thresh=0.1, flag=BAD)
+ flagscol = flags_result[field]
assert np.all(flagscol[expected] == BAD)
def test_constants_flagVarianceBased(data):
expected = np.arange(5, 25)
field, *_ = data.columns
- flagger = initFlagsLike(data)
- data, flagger_result1 = flagByVariance(data, field, flagger, window="1h", flag=BAD)
+ flags = initFlagsLike(data)
+ data, flags_result1 = flagByVariance(data, field, flags, window="1h", flag=BAD)
- flag_result1 = flagger_result1[field]
+ flag_result1 = flags_result1[field]
test_sum = (flag_result1[expected] == BAD).sum()
assert test_sum == len(expected)
diff --git a/tests/funcs/test_functions.py b/tests/funcs/test_functions.py
index 06eef82da..7d625d71e 100644
--- a/tests/funcs/test_functions.py
+++ b/tests/funcs/test_functions.py
@@ -28,9 +28,9 @@ def field(data):
def test_flagRange(data, field):
min, max = 10, 90
- flagger = initFlagsLike(data)
- data, flagger = flagRange(data, field, flagger, min=min, max=max, flag=BAD)
- flagged = flagger[field] > UNFLAGGED
+ flags = initFlagsLike(data)
+ data, flags = flagRange(data, field, flags, min=min, max=max, flag=BAD)
+ flagged = flags[field] > UNFLAGGED
expected = (data[field] < min) | (data[field] > max)
assert all(flagged == expected)
@@ -47,47 +47,47 @@ def test_flagSesonalRange(data, field):
]
for test, expected in tests:
- flagger = initFlagsLike(data)
+ flags = initFlagsLike(data)
newfield = f"{field}_masked"
start = f"{test['startmonth']:02}-{test['startday']:02}T00:00:00"
end = f"{test['endmonth']:02}-{test['endday']:02}T00:00:00"
- data, flagger = copy(data, field, flagger, field + "_masked")
- data, flagger = mask(
- data, newfield, flagger,
+ data, flags = copy(data, field, flags, field + "_masked")
+ data, flags = mask(
+ data, newfield, flags,
mode='periodic', period_start=start, period_end=end, include_bounds=True, flag=BAD
)
- data, flagger = flagRange(data, newfield, flagger, min=test['min'], max=test['max'], flag=BAD)
- data, flagger = reindexFlags(data, field, flagger, method='match', source=newfield, flag=BAD)
- data, flagger = drop(data, newfield, flagger)
- flagged = flagger[field] > UNFLAGGED
+ data, flags = flagRange(data, newfield, flags, min=test['min'], max=test['max'], flag=BAD)
+ data, flags = reindexFlags(data, field, flags, method='match', source=newfield, flag=BAD)
+ data, flags = drop(data, newfield, flags)
+ flagged = flags[field] > UNFLAGGED
assert flagged.sum() == expected
def test_clearFlags(data, field):
- flagger = initFlagsLike(data)
- flagger[:, field] = BAD
- assert all(flagger[field] == BAD)
+ flags = initFlagsLike(data)
+ flags[:, field] = BAD
+ assert all(flags[field] == BAD)
- _, flagger = clearFlags(data, field, flagger)
- assert all(flagger[field] == UNFLAGGED)
+ _, flags = clearFlags(data, field, flags)
+ assert all(flags[field] == UNFLAGGED)
def test_forceFlags(data, field):
- flagger = initFlagsLike(data)
- flagger[:, field] = BAD
- assert all(flagger[field] == BAD)
+ flags = initFlagsLike(data)
+ flags[:, field] = BAD
+ assert all(flags[field] == BAD)
- _, flagger = forceFlags(data, field, flagger, flag=DOUBT)
- assert all(flagger[field] == DOUBT)
+ _, flags = forceFlags(data, field, flags, flag=DOUBT)
+ assert all(flags[field] == DOUBT)
def test_flagIsolated(data, field):
- flagger = initFlagsLike(data)
+ flags = initFlagsLike(data)
data.iloc[1:3, 0] = np.nan
data.iloc[4:5, 0] = np.nan
- flagger[data[field].index[5:6], field] = BAD
+ flags[data[field].index[5:6], field] = BAD
data.iloc[11:13, 0] = np.nan
data.iloc[15:17, 0] = np.nan
@@ -102,15 +102,15 @@ def test_flagIsolated(data, field):
# 2016-01-08 7.0 -inf
# .. .. ..
- _, flagger_result = flagIsolated(data, field, flagger, group_window="1D", gap_window="2.1D", flag=BAD)
+ _, flags_result = flagIsolated(data, field, flags, group_window="1D", gap_window="2.1D", flag=BAD)
- assert flagger_result[field].iloc[[3, 5]].all()
+ assert flags_result[field].iloc[[3, 5]].all()
- data, flagger_result = flagIsolated(
- data, field, flagger_result,
+ data, flags_result = flagIsolated(
+ data, field, flags_result,
group_window="2D", gap_window="2.1D", continuation_range="1.1D", flag=BAD
)
- assert flagger_result[field].iloc[[3, 5, 13, 14]].all()
+ assert flags_result[field].iloc[[3, 5, 13, 14]].all()
@pytest.mark.parametrize("dat", [pytest.lazy_fixture("course_2")])
@@ -123,16 +123,16 @@ def test_flagCrossScoring(dat):
s1 = pd.Series(data=s1.values, index=s1.index)
s2 = pd.Series(data=s2.values, index=s1.index)
data = dios.DictOfSeries([s1, s2], columns=["data1", "data2"])
- flagger = initFlagsLike(data)
- _, flagger_result = flagCrossStatistic(data, field, flagger, fields=fields, thresh=3, cross_stat=np.mean, flag=BAD)
+ flags = initFlagsLike(data)
+ _, flags_result = flagCrossStatistic(data, field, flags, fields=fields, thresh=3, cross_stat=np.mean, flag=BAD)
for field in fields:
- isflagged = flagger_result[field] > UNFLAGGED
+ isflagged = flags_result[field] > UNFLAGGED
assert isflagged[characteristics["raise"]].all()
def test_flagManual(data, field):
- flagger = initFlagsLike(data)
- args = data, field, flagger
+ flags = initFlagsLike(data)
+ args = data, field, flags
dat = data[field]
mdata = pd.Series("lala", index=dat.index)
@@ -220,31 +220,31 @@ def test_flagDriftFromNormal(dat):
data['d4'] = 3 + 4 * data['d1']
data['d5'] = 3 + 4 * data['d1']
- flagger = initFlagsLike(data)
- data_norm, flagger_norm = flagDriftFromNorm(
- data, 'dummy', flagger,
+ flags = initFlagsLike(data)
+ data_norm, flags_norm = flagDriftFromNorm(
+ data, 'dummy', flags,
['d1', 'd2', 'd3'],
segment_freq="200min",
norm_spread=5,
flag=BAD,
)
- data_ref, flagger_ref = flagDriftFromReference(
- data, 'd1', flagger,
+ data_ref, flags_ref = flagDriftFromReference(
+ data, 'd1', flags,
['d1', 'd2', 'd3'],
segment_freq="3D",
thresh=20,
flag=BAD,
)
- data_scale, flagger_scale = flagDriftFromScaledNorm(
- data, 'dummy', flagger,
+ data_scale, flags_scale = flagDriftFromScaledNorm(
+ data, 'dummy', flags,
['d1', 'd3'], ['d4', 'd5'],
segment_freq="3D",
thresh=20,
norm_spread=5,
flag=BAD,
)
- assert all(flagger_norm['d3'] > UNFLAGGED)
- assert all(flagger_ref['d3'] > UNFLAGGED)
- assert all(flagger_scale['d3'] > UNFLAGGED)
+ assert all(flags_norm['d3'] > UNFLAGGED)
+ assert all(flags_ref['d3'] > UNFLAGGED)
+ assert all(flags_scale['d3'] > UNFLAGGED)
diff --git a/tests/funcs/test_generic_api_functions.py b/tests/funcs/test_generic_api_functions.py
index 8c3ce15ff..64b922fae 100644
--- a/tests/funcs/test_generic_api_functions.py
+++ b/tests/funcs/test_generic_api_functions.py
@@ -24,19 +24,19 @@ def test_addFieldFlagGeneric(data):
saqc = SaQC(data=data)
func = lambda var1: pd.Series(False, index=data[var1.name].index)
- data, flagger = saqc.generic.flag("tmp1", func, flag=BAD).getResult()
- assert "tmp1" in flagger.columns and "tmp1" not in data
+ data, flags = saqc.generic.flag("tmp1", func, flag=BAD).getResult()
+ assert "tmp1" in flags.columns and "tmp1" not in data
def test_addFieldProcGeneric(data):
saqc = SaQC(data=data)
func = lambda: pd.Series([])
- data, flagger = saqc.generic.process("tmp1", func, flag=BAD ).getResult(raw=True)
+ data, flags = saqc.generic.process("tmp1", func, flag=BAD ).getResult(raw=True)
assert "tmp1" in data.columns and data["tmp1"].empty
func = lambda var1, var2: var1 + var2
- data, flagger = saqc.generic.process("tmp2", func, flag=BAD).getResult()
+ data, flags = saqc.generic.process("tmp2", func, flag=BAD).getResult()
assert "tmp2" in data.columns and (data["tmp2"] == data["var1"] + data["var2"]).all(axis=None)
@@ -48,6 +48,6 @@ def test_mask(data):
data, _ = saqc.generic.process("var1", lambda var1: mask(var1 < mean), flag=BAD).getResult()
assert ((data["var1"].isna()) == (data_org["var1"] < 10) & data_org["var1"].isna()).all(axis=None)
- data, flagger = saqc.generic.process("tmp", lambda var1: mask(var1 < mean), flag=BAD).getResult()
- assert ("tmp" in data.columns) and ("tmp" in flagger.columns)
+ data, flags = saqc.generic.process("tmp", lambda var1: mask(var1 < mean), flag=BAD).getResult()
+ assert ("tmp" in data.columns) and ("tmp" in flags.columns)
assert ((data["tmp"].isna()) == (data_org["var1"] < 10) & data_org["var1"].isna()).all(axis=None)
diff --git a/tests/funcs/test_generic_config_functions.py b/tests/funcs/test_generic_config_functions.py
index 5b3a28cb5..a47407866 100644
--- a/tests/funcs/test_generic_config_functions.py
+++ b/tests/funcs/test_generic_config_functions.py
@@ -8,7 +8,7 @@ import pandas as pd
import dios
from saqc.constants import *
-from saqc.core import initFlagsLike, Flags as Flagger
+from saqc.core import initFlagsLike, Flags
from saqc.core.visitor import ConfigFunctionParser
from saqc.core.config import Fields as F
from saqc.core.register import register
@@ -33,14 +33,14 @@ def data_diff():
return dios.DictOfSeries(data={col0.name: col0.iloc[: mid + offset], col1.name: col1.iloc[mid - offset :],})
-def _compileGeneric(expr, flagger):
+def _compileGeneric(expr, flags):
tree = ast.parse(expr, mode="eval")
- _, kwargs = ConfigFunctionParser(flagger).parse(tree.body)
+ _, kwargs = ConfigFunctionParser(flags).parse(tree.body)
return kwargs["func"]
def test_missingIdentifier(data):
- flagger = Flagger()
+ flags = Flags()
# NOTE:
# - the error is only raised at runtime during parsing would be better
@@ -50,13 +50,13 @@ def test_missingIdentifier(data):
]
for test in tests:
- func = _compileGeneric(f"generic.flag(func={test})", flagger)
+ func = _compileGeneric(f"generic.flag(func={test})", flags)
with pytest.raises(NameError):
- _execGeneric(flagger, data, func, field="", nodata=np.nan)
+ _execGeneric(flags, data, func, field="", nodata=np.nan)
def test_syntaxError():
- flagger = Flagger()
+ flags = Flags()
tests = [
"range(x=5",
"rangex=5)",
@@ -65,7 +65,7 @@ def test_syntaxError():
for test in tests:
with pytest.raises(SyntaxError):
- _compileGeneric(f"flag(func={test})", flagger)
+ _compileGeneric(f"flag(func={test})", flags)
def test_typeError():
@@ -73,18 +73,18 @@ def test_typeError():
test that forbidden constructs actually throw an error
TODO: find a few more cases or get rid of the test
"""
- flagger = Flagger()
+ flags = Flags()
# : think about cases that should be forbidden
tests = ("lambda x: x * 2",)
for test in tests:
with pytest.raises(TypeError):
- _compileGeneric(f"generic.flag(func={test})", flagger)
+ _compileGeneric(f"generic.flag(func={test})", flags)
def test_comparisonOperators(data):
- flagger = initFlagsLike(data)
+ flags = initFlagsLike(data)
var1, var2, *_ = data.columns
this = var1
@@ -98,13 +98,13 @@ def test_comparisonOperators(data):
]
for test, expected in tests:
- func = _compileGeneric(f"generic.flag(func={test})", flagger)
- result = _execGeneric(flagger, data, func, field=var1, nodata=np.nan)
+ func = _compileGeneric(f"generic.flag(func={test})", flags)
+ result = _execGeneric(flags, data, func, field=var1, nodata=np.nan)
assert np.all(result == expected)
def test_arithmeticOperators(data):
- flagger = initFlagsLike(data)
+ flags = initFlagsLike(data)
var1, *_ = data.columns
this = data[var1]
@@ -118,13 +118,13 @@ def test_arithmeticOperators(data):
]
for test, expected in tests:
- func = _compileGeneric(f"generic.process(func={test})", flagger)
- result = _execGeneric(flagger, data, func, field=var1, nodata=np.nan)
+ func = _compileGeneric(f"generic.process(func={test})", flags)
+ result = _execGeneric(flags, data, func, field=var1, nodata=np.nan)
assert np.all(result == expected)
def test_nonReduncingBuiltins(data):
- flagger = initFlagsLike(data)
+ flags = initFlagsLike(data)
var1, *_ = data.columns
this = var1
mean = data[var1].mean()
@@ -137,15 +137,15 @@ def test_nonReduncingBuiltins(data):
]
for test, expected in tests:
- func = _compileGeneric(f"generic.process(func={test})", flagger)
- result = _execGeneric(flagger, data, func, field=this, nodata=np.nan)
+ func = _compileGeneric(f"generic.process(func={test})", flags)
+ result = _execGeneric(flags, data, func, field=this, nodata=np.nan)
assert (result == expected).all()
@pytest.mark.parametrize("nodata", TESTNODATA)
def test_reduncingBuiltins(data, nodata):
data.loc[::4] = nodata
- flagger = initFlagsLike(data)
+ flags = initFlagsLike(data)
var1 = data.columns[0]
this = data.iloc[:, 0]
@@ -159,15 +159,15 @@ def test_reduncingBuiltins(data, nodata):
]
for test, expected in tests:
- func = _compileGeneric(f"generic.process(func={test})", flagger)
- result = _execGeneric(flagger, data, func, field=this.name, nodata=nodata)
+ func = _compileGeneric(f"generic.process(func={test})", flags)
+ result = _execGeneric(flags, data, func, field=this.name, nodata=nodata)
assert result == expected
@pytest.mark.parametrize("nodata", TESTNODATA)
def test_ismissing(data, nodata):
- flagger = initFlagsLike(data)
+ flags = initFlagsLike(data)
data.iloc[: len(data) // 2, 0] = np.nan
data.iloc[(len(data) // 2) + 1 :, 0] = -9999
this = data.iloc[:, 0]
@@ -178,8 +178,8 @@ def test_ismissing(data, nodata):
]
for test, expected in tests:
- func = _compileGeneric(f"generic.flag(func={test})", flagger)
- result = _execGeneric(flagger, data, func, this.name, nodata)
+ func = _compileGeneric(f"generic.flag(func={test})", flags)
+ result = _execGeneric(flags, data, func, this.name, nodata)
assert np.all(result == expected)
@@ -188,7 +188,7 @@ def test_bitOps(data, nodata):
var1, var2, *_ = data.columns
this = var1
- flagger = initFlagsLike(data)
+ flags = initFlagsLike(data)
tests = [
("~(this > mean(this))", ~(data[this] > np.nanmean(data[this]))),
@@ -197,29 +197,29 @@ def test_bitOps(data, nodata):
]
for test, expected in tests:
- func = _compileGeneric(f"generic.flag(func={test})", flagger)
- result = _execGeneric(flagger, data, func, this, nodata)
+ func = _compileGeneric(f"generic.flag(func={test})", flags)
+ result = _execGeneric(flags, data, func, this, nodata)
assert np.all(result == expected)
def test_isflagged(data):
var1, var2, *_ = data.columns
- flagger = initFlagsLike(data)
- flagger[data[var1].index[::2], var1] = BAD
+ flags = initFlagsLike(data)
+ flags[data[var1].index[::2], var1] = BAD
tests = [
- (f"isflagged({var1})", flagger[var1] > UNFLAGGED),
- (f"isflagged({var1}, flag=BAD)", flagger[var1] >= BAD),
- (f"isflagged({var1}, UNFLAGGED, '==')", flagger[var1] == UNFLAGGED),
- (f"~isflagged({var2})", flagger[var2] == UNFLAGGED),
- (f"~({var2}>999) & (~isflagged({var2}))", ~(data[var2] > 999) & (flagger[var2] == UNFLAGGED)),
+ (f"isflagged({var1})", flags[var1] > UNFLAGGED),
+ (f"isflagged({var1}, flag=BAD)", flags[var1] >= BAD),
+ (f"isflagged({var1}, UNFLAGGED, '==')", flags[var1] == UNFLAGGED),
+ (f"~isflagged({var2})", flags[var2] == UNFLAGGED),
+ (f"~({var2}>999) & (~isflagged({var2}))", ~(data[var2] > 999) & (flags[var2] == UNFLAGGED)),
]
for i, (test, expected) in enumerate(tests):
try:
- func = _compileGeneric(f"generic.flag(func={test}, flag=BAD)", flagger)
- result = _execGeneric(flagger, data, func, field=None, nodata=np.nan)
+ func = _compileGeneric(f"generic.flag(func={test}, flag=BAD)", flags)
+ result = _execGeneric(flags, data, func, field=None, nodata=np.nan)
assert np.all(result == expected)
except Exception:
print(i, test)
@@ -229,9 +229,9 @@ def test_isflagged(data):
for comp in ['>', '>=', '==', '!=', '<', '<=']:
fails = f"isflagged({var1}, comparator='{comp}')"
- func = _compileGeneric(f"generic.flag(func={fails}, flag=BAD)", flagger)
+ func = _compileGeneric(f"generic.flag(func={fails}, flag=BAD)", flags)
with pytest.raises(ValueError):
- _execGeneric(flagger, data, func, field=None, nodata=np.nan)
+ _execGeneric(flags, data, func, field=None, nodata=np.nan)
def test_variableAssignments(data):
@@ -245,12 +245,12 @@ def test_variableAssignments(data):
fobj = writeIO(config)
saqc = SaQC(data).readConfig(fobj)
- result_data, result_flagger = saqc.getResult(raw=True)
+ result_data, result_flags = saqc.getResult(raw=True)
assert set(result_data.columns) == set(data.columns) | {
"dummy1",
}
- assert set(result_flagger.columns) == set(data.columns) | {"dummy1", "dummy2"}
+ assert set(result_flags.columns) == set(data.columns) | {"dummy1", "dummy2"}
# TODO: why this must(!) fail ? - a comment would be helpful
@@ -266,8 +266,8 @@ def test_processMultiple(data_diff):
fobj = writeIO(config)
saqc = SaQC(data_diff).readConfig(fobj)
- result_data, result_flagger = saqc.getResult()
- assert len(result_data["dummy"]) == len(result_flagger["dummy"])
+ result_data, result_flags = saqc.getResult()
+ assert len(result_data["dummy"]) == len(result_flags["dummy"])
def test_callableArgumentsUnary(data):
@@ -275,7 +275,7 @@ def test_callableArgumentsUnary(data):
window = 5
@register(masking='field')
- def testFuncUnary(data, field, flagger, func, **kwargs):
+ def testFuncUnary(data, field, flags, func, **kwargs):
data[field] = data[field].rolling(window=window).apply(func)
return data, initFlagsLike(data)
@@ -304,7 +304,7 @@ def test_callableArgumentsBinary(data):
var1, var2 = data.columns[:2]
@register(masking='field')
- def testFuncBinary(data, field, flagger, func, **kwargs):
+ def testFuncBinary(data, field, flags, func, **kwargs):
data[field] = func(data[var1], data[var2])
return data, initFlagsLike(data)
diff --git a/tests/funcs/test_harm_funcs.py b/tests/funcs/test_harm_funcs.py
index 3f0140bd6..a8606ca72 100644
--- a/tests/funcs/test_harm_funcs.py
+++ b/tests/funcs/test_harm_funcs.py
@@ -6,7 +6,7 @@ import numpy as np
import pandas as pd
import dios
-from saqc.core import initFlagsLike, Flags as Flagger
+from saqc.core import initFlagsLike, Flags
from saqc.constants import BAD, UNFLAGGED
from saqc.funcs.resampling import (
linear,
@@ -16,7 +16,7 @@ from saqc.funcs.resampling import (
mapToOriginal,
)
-from tests.common import checkDataFlaggerInvariants
+from tests.common import checkDataFlagsInvariants
@pytest.fixture
@@ -44,14 +44,14 @@ def data():
def test_wrapper(data, func, kws):
field = 'data'
freq = "15min"
- flagger = initFlagsLike(data)
+ flags = initFlagsLike(data)
import saqc
func = getattr(saqc.funcs, func)
- data, flagger = func(data, field, flagger, freq, **kws)
+ data, flags = func(data, field, flags, freq, **kws)
# check minimal requirements
- checkDataFlaggerInvariants(data, flagger, field)
+ checkDataFlagsInvariants(data, flags, field)
assert data[field].index.freq == pd.Timedelta(freq)
@@ -62,18 +62,18 @@ def test_gridInterpolation(data, method):
data = data[field]
data = (data * np.sin(data)).append(data.shift(1, "2h")).shift(1, "3s")
data = dios.DictOfSeries(data)
- flagger = initFlagsLike(data)
+ flags = initFlagsLike(data)
# we are just testing if the interpolation gets passed to the series without causing an error:
- res = interpolate(data, field, flagger, freq, method=method, downcast_interpolation=True)
+ res = interpolate(data, field, flags, freq, method=method, downcast_interpolation=True)
if method == "polynomial":
- res = interpolate(data, field, flagger, freq, order=2, method=method, downcast_interpolation=True)
- res = interpolate(data, field, flagger, freq, order=10, method=method, downcast_interpolation=True)
+ res = interpolate(data, field, flags, freq, order=2, method=method, downcast_interpolation=True)
+ res = interpolate(data, field, flags, freq, order=10, method=method, downcast_interpolation=True)
# check minimal requirements
- rdata, rflagger = res
- checkDataFlaggerInvariants(rdata, rflagger, field, identical=False)
+ rdata, rflags = res
+ checkDataFlagsInvariants(rdata, rflags, field, identical=False)
assert rdata[field].index.freq == pd.Timedelta(freq)
@@ -105,23 +105,23 @@ def test_flagsSurviveBackprojection():
@pytest.mark.parametrize("reshaper", ["nshift", "fshift", "bshift", "nagg", "bagg", "fagg", "interpolation"])
def test_harmSingleVarIntermediateFlagging(data, reshaper):
- flagger = initFlagsLike(data)
+ flags = initFlagsLike(data)
field = 'data'
pre_data = data.copy()
- pre_flagger = flagger.copy()
+ pre_flags = flags.copy()
- data, flagger = linear(data, field, flagger, freq="15min")
- checkDataFlaggerInvariants(data, flagger, field, identical=True)
+ data, flags = linear(data, field, flags, freq="15min")
+ checkDataFlagsInvariants(data, flags, field, identical=True)
assert data[field].index.freq == pd.Timedelta('15min')
# flag something bad
- flagger[data[field].index[3:4], field] = BAD
- data, flagger = mapToOriginal(data, field, flagger, method="inverse_" + reshaper)
+ flags[data[field].index[3:4], field] = BAD
+ data, flags = mapToOriginal(data, field, flags, method="inverse_" + reshaper)
- assert len(data[field]) == len(flagger[field])
+ assert len(data[field]) == len(flags[field])
assert data[field].equals(pre_data[field])
- assert flagger[field].index.equals(pre_flagger[field].index)
+ assert flags[field].index.equals(pre_flags[field].index)
if 'agg' in reshaper:
if reshaper == "nagg":
@@ -133,9 +133,9 @@ def test_harmSingleVarIntermediateFlagging(data, reshaper):
else:
raise NotImplementedError('untested test case')
- assert all(flagger[field].iloc[start:end] > UNFLAGGED)
- assert all(flagger[field].iloc[:start] == UNFLAGGED)
- assert all(flagger[field].iloc[end:] == UNFLAGGED)
+ assert all(flags[field].iloc[start:end] > UNFLAGGED)
+ assert all(flags[field].iloc[:start] == UNFLAGGED)
+ assert all(flags[field].iloc[end:] == UNFLAGGED)
elif 'shift' in reshaper:
if reshaper == "nshift":
@@ -147,7 +147,7 @@ def test_harmSingleVarIntermediateFlagging(data, reshaper):
else:
raise NotImplementedError('untested test case')
- flagged = flagger[field] > UNFLAGGED
+ flagged = flags[field] > UNFLAGGED
assert all(flagged == exp)
elif reshaper == 'interpolation':
@@ -166,22 +166,22 @@ def test_harmSingleVarIntermediateFlagging(data, reshaper):
(("bagg", "30Min"), pd.Series(data=[-50.0, -75.0, 50.0, 50.0], index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30min"))),
])
def test_harmSingleVarInterpolationAgg(data, params, expected):
- flagger = initFlagsLike(data)
+ flags = initFlagsLike(data)
field = 'data'
pre_data = data.copy()
- pre_flaggger = flagger.copy()
+ pre_flaggger = flags.copy()
method, freq = params
- data_harm, flagger_harm = aggregate(data, field, flagger, freq, value_func=np.sum, method=method)
- checkDataFlaggerInvariants(data_harm, flagger_harm, field, identical=True)
+ data_harm, flags_harm = aggregate(data, field, flags, freq, value_func=np.sum, method=method)
+ checkDataFlagsInvariants(data_harm, flags_harm, field, identical=True)
assert data_harm[field].index.freq == pd.Timedelta(freq)
assert data_harm[field].equals(expected)
- data_deharm, flagger_deharm = mapToOriginal(data_harm, "data", flagger_harm, method="inverse_" + method)
- checkDataFlaggerInvariants(data_harm, flagger_harm, field, identical=True)
+ data_deharm, flags_deharm = mapToOriginal(data_harm, "data", flags_harm, method="inverse_" + method)
+ checkDataFlagsInvariants(data_harm, flags_harm, field, identical=True)
assert data_deharm[field].equals(pre_data[field])
- assert flagger_deharm[field].equals(pre_flaggger[field])
+ assert flags_deharm[field].equals(pre_flaggger[field])
@pytest.mark.parametrize(
@@ -195,17 +195,17 @@ def test_harmSingleVarInterpolationAgg(data, params, expected):
(("nshift", "30min"), pd.Series(data=[np.nan, -37.5, 12.5, 50.0], index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"))),
])
def test_harmSingleVarInterpolationShift(data, params, expected):
- flagger = initFlagsLike(data)
+ flags = initFlagsLike(data)
field = 'data'
pre_data = data.copy()
- pre_flagger = flagger.copy()
+ pre_flags = flags.copy()
method, freq = params
- data_harm, flagger_harm = shift(data, field, flagger, freq, method=method)
+ data_harm, flags_harm = shift(data, field, flags, freq, method=method)
assert data_harm[field].equals(expected)
- data_deharm, flagger_deharm = mapToOriginal(data_harm, "data", flagger_harm, method="inverse_" + method)
+ data_deharm, flags_deharm = mapToOriginal(data_harm, "data", flags_harm, method="inverse_" + method)
assert data_deharm[field].equals(pre_data[field])
- assert flagger_deharm[field].equals(pre_flagger[field])
+ assert flags_deharm[field].equals(pre_flags[field])
diff --git a/tests/funcs/test_modelling.py b/tests/funcs/test_modelling.py
index de9f1efb8..5bfdfba88 100644
--- a/tests/funcs/test_modelling.py
+++ b/tests/funcs/test_modelling.py
@@ -20,16 +20,16 @@ def test_modelling_polyFit_forRegular(dat):
# add some nice sine distortion
data = data + 10 * np.sin(np.arange(0, len(data.indexes[0])))
data = dios.DictOfSeries(data)
- flagger = initFlagsLike(data)
- result1, _ = calculatePolynomialResidues(data, "data", flagger, 11, 2, numba=False)
- result2, _ = calculatePolynomialResidues(data, "data", flagger, 11, 2, numba=True)
+ flags = initFlagsLike(data)
+ result1, _ = calculatePolynomialResidues(data, "data", flags, 11, 2, numba=False)
+ result2, _ = calculatePolynomialResidues(data, "data", flags, 11, 2, numba=True)
assert (result1["data"] - result2["data"]).abs().max() < 10 ** -10
- result3, _ = calculatePolynomialResidues(data, "data", flagger, "110min", 2, numba=False)
+ result3, _ = calculatePolynomialResidues(data, "data", flags, "110min", 2, numba=False)
assert result3["data"].equals(result1["data"])
- result4, _ = calculatePolynomialResidues(data, "data", flagger, 11, 2, numba=True, min_periods=11)
+ result4, _ = calculatePolynomialResidues(data, "data", flags, 11, 2, numba=True, min_periods=11)
assert (result4["data"] - result2["data"]).abs().max() < 10 ** -10
data.iloc[13:16] = np.nan
- result5, _ = calculatePolynomialResidues(data, "data", flagger, 11, 2, numba=True, min_periods=9)
+ result5, _ = calculatePolynomialResidues(data, "data", flags, 11, 2, numba=True, min_periods=9)
assert result5["data"].iloc[10:19].isna().all()
@@ -37,45 +37,45 @@ def test_modelling_polyFit_forRegular(dat):
def test_modelling_rollingMean_forRegular(dat):
data, _ = dat(freq="10min", periods=30, initial_level=0, final_level=100, out_val=-100)
data = dios.DictOfSeries(data)
- flagger = initFlagsLike(data)
- calculateRollingResidues(data, "data", flagger, 5, func=np.mean, eval_flags=True, min_periods=0, center=True)
- calculateRollingResidues(data, "data", flagger, 5, func=np.mean, eval_flags=True, min_periods=0, center=False)
+ flags = initFlagsLike(data)
+ calculateRollingResidues(data, "data", flags, 5, func=np.mean, eval_flags=True, min_periods=0, center=True)
+ calculateRollingResidues(data, "data", flags, 5, func=np.mean, eval_flags=True, min_periods=0, center=False)
@pytest.mark.parametrize("dat", [pytest.lazy_fixture("course_1")])
def test_modelling_mask(dat):
data, _ = dat()
data = dios.DictOfSeries(data)
- flagger = initFlagsLike(data)
+ flags = initFlagsLike(data)
field = "data"
# set flags everywhere to test unflagging
- flagger[:, field] = BAD
+ flags[:, field] = BAD
- common = dict(data=data, field=field, flagger=flagger, mode='periodic')
- data_seasonal, flagger_seasonal = mask(**common, period_start="20:00", period_end="40:00", include_bounds=False)
- flagscol = flagger_seasonal[field]
+ common = dict(data=data, field=field, flags=flags, mode='periodic')
+ data_seasonal, flags_seasonal = mask(**common, period_start="20:00", period_end="40:00", include_bounds=False)
+ flagscol = flags_seasonal[field]
m = (20 <= flagscol.index.minute) & (flagscol.index.minute <= 40)
- assert all(flagger_seasonal[field][m] == UNFLAGGED)
+ assert all(flags_seasonal[field][m] == UNFLAGGED)
assert all(data_seasonal[field][m].isna())
- data_seasonal, flagger_seasonal = mask(**common, period_start="15:00:00", period_end="02:00:00")
- flagscol = flagger_seasonal[field]
+ data_seasonal, flags_seasonal = mask(**common, period_start="15:00:00", period_end="02:00:00")
+ flagscol = flags_seasonal[field]
m = (15 <= flagscol.index.hour) & (flagscol.index.hour <= 2)
- assert all(flagger_seasonal[field][m] == UNFLAGGED)
+ assert all(flags_seasonal[field][m] == UNFLAGGED)
assert all(data_seasonal[field][m].isna())
- data_seasonal, flagger_seasonal = mask(**common, period_start="03T00:00:00", period_end="10T00:00:00")
- flagscol = flagger_seasonal[field]
+ data_seasonal, flags_seasonal = mask(**common, period_start="03T00:00:00", period_end="10T00:00:00")
+ flagscol = flags_seasonal[field]
m = (3 <= flagscol.index.hour) & (flagscol.index.hour <= 10)
- assert all(flagger_seasonal[field][m] == UNFLAGGED)
+ assert all(flags_seasonal[field][m] == UNFLAGGED)
assert all(data_seasonal[field][m].isna())
mask_ser = pd.Series(False, index=data["data"].index)
mask_ser[::5] = True
data["mask_ser"] = mask_ser
- flagger = initFlagsLike(data)
- data_masked, flagger_masked = mask(data, "data", flagger, mode='mask_var', mask_var="mask_ser")
+ flags = initFlagsLike(data)
+ data_masked, flags_masked = mask(data, "data", flags, mode='mask_var', mask_var="mask_ser")
m = mask_ser
- assert all(flagger_masked[field][m] == UNFLAGGED)
+ assert all(flags_masked[field][m] == UNFLAGGED)
assert all(data_masked[field][m].isna())
diff --git a/tests/funcs/test_pattern_rec.py b/tests/funcs/test_pattern_rec.py
index db3c50249..1cd7b7b4d 100644
--- a/tests/funcs/test_pattern_rec.py
+++ b/tests/funcs/test_pattern_rec.py
@@ -28,12 +28,12 @@ def test_flagPattern_wavelet():
pattern = data.iloc[1:6]
data = dios.DictOfSeries(dict(data=data, pattern_data=pattern))
- flagger = initFlagsLike(data, name='data')
- data, flagger = flagPatternByDTW(data, "data", flagger, ref_field="pattern_data", flag=BAD)
+ flags = initFlagsLike(data, name='data')
+ data, flags = flagPatternByDTW(data, "data", flags, ref_field="pattern_data", flag=BAD)
- assert all(flagger["data"][1:6])
- assert any(flagger["data"][:1])
- assert any(flagger["data"][7:])
+ assert all(flags["data"][1:6])
+ assert any(flags["data"][:1])
+ assert any(flags["data"][7:])
@pytest.mark.skip(reason='faulty implementation - will get fixed by GL-MR191')
@@ -43,9 +43,9 @@ def test_flagPattern_dtw():
pattern = data.iloc[1:6]
data = dios.DictOfSeries(dict(data=data, pattern_data=pattern))
- flagger = initFlagsLike(data, name='data')
- data, flagger = flagPatternByWavelet(data, "data", flagger, ref_field="pattern_data", flag=BAD)
+ flags = initFlagsLike(data, name='data')
+ data, flags = flagPatternByWavelet(data, "data", flags, ref_field="pattern_data", flag=BAD)
- assert all(flagger["data"][1:6])
- assert any(flagger["data"][:1])
- assert any(flagger["data"][7:])
+ assert all(flags["data"][1:6])
+ assert any(flags["data"][:1])
+ assert any(flags["data"][7:])
diff --git a/tests/funcs/test_proc_functions.py b/tests/funcs/test_proc_functions.py
index 04739b40b..cfcd5bcf6 100644
--- a/tests/funcs/test_proc_functions.py
+++ b/tests/funcs/test_proc_functions.py
@@ -21,15 +21,15 @@ def test_rollingInterpolateMissing(course_5):
data, characteristics = course_5(periods=10, nan_slice=[5, 6])
field = data.columns[0]
data = dios.DictOfSeries(data)
- flagger = initFlagsLike(data)
+ flags = initFlagsLike(data)
dataInt, *_ = interpolateByRolling(
- data, field, flagger, 3, func=np.median, center=True, min_periods=0, interpol_flag=UNFLAGGED
+ data, field, flags, 3, func=np.median, center=True, min_periods=0, interpol_flag=UNFLAGGED
)
# import pdb
# pdb.set_trace()
assert dataInt[field][characteristics["missing"]].notna().all()
dataInt, *_ = interpolateByRolling(
- data, field, flagger, 3, func=np.nanmean, center=False, min_periods=3, interpol_flag=UNFLAGGED
+ data, field, flags, 3, func=np.nanmean, center=False, min_periods=3, interpol_flag=UNFLAGGED
)
assert dataInt[field][characteristics["missing"]].isna().all()
@@ -38,15 +38,15 @@ def test_interpolateMissing(course_5):
data, characteristics = course_5(periods=10, nan_slice=[5])
field = data.columns[0]
data = dios.DictOfSeries(data)
- flagger = initFlagsLike(data)
- dataLin, *_ = interpolateInvalid(data, field, flagger, method="linear")
- dataPoly, *_ = interpolateInvalid(data, field, flagger, method="polynomial")
+ flags = initFlagsLike(data)
+ dataLin, *_ = interpolateInvalid(data, field, flags, method="linear")
+ dataPoly, *_ = interpolateInvalid(data, field, flags, method="polynomial")
assert dataLin[field][characteristics["missing"]].notna().all()
assert dataPoly[field][characteristics["missing"]].notna().all()
data, characteristics = course_5(periods=10, nan_slice=[5, 6, 7])
- dataLin1, *_ = interpolateInvalid(data, field, flagger, method="linear", inter_limit=2)
- dataLin2, *_ = interpolateInvalid(data, field, flagger, method="linear", inter_limit=3)
- dataLin3, *_ = interpolateInvalid(data, field, flagger, method="linear", inter_limit=4)
+ dataLin1, *_ = interpolateInvalid(data, field, flags, method="linear", inter_limit=2)
+ dataLin2, *_ = interpolateInvalid(data, field, flags, method="linear", inter_limit=3)
+ dataLin3, *_ = interpolateInvalid(data, field, flags, method="linear", inter_limit=4)
assert dataLin1[field][characteristics["missing"]].isna().all()
assert dataLin2[field][characteristics["missing"]].isna().all()
assert dataLin3[field][characteristics["missing"]].notna().all()
@@ -56,13 +56,13 @@ def test_transform(course_5):
data, characteristics = course_5(periods=10, nan_slice=[5, 6])
field = data.columns[0]
data = dios.DictOfSeries(data)
- flagger = initFlagsLike(data)
- data1, *_ = transform(data, field, flagger, func=linearInterpolation)
+ flags = initFlagsLike(data)
+ data1, *_ = transform(data, field, flags, func=linearInterpolation)
assert data1[field][characteristics["missing"]].isna().all()
- data1, *_ = transform(data, field, flagger, func=lambda x: linearInterpolation(x, inter_limit=3))
+ data1, *_ = transform(data, field, flags, func=lambda x: linearInterpolation(x, inter_limit=3))
assert data1[field][characteristics["missing"]].notna().all()
data1, *_ = transform(
- data, field, flagger, func=lambda x: polynomialInterpolation(x, inter_limit=3, inter_order=3)
+ data, field, flags, func=lambda x: polynomialInterpolation(x, inter_limit=3, inter_order=3)
)
assert data1[field][characteristics["missing"]].notna().all()
@@ -71,8 +71,8 @@ def test_resample(course_5):
data, characteristics = course_5(freq="1min", periods=30, nan_slice=[1, 11, 12, 22, 24, 26])
field = data.columns[0]
data = dios.DictOfSeries(data)
- flagger = initFlagsLike(data)
- data1, *_ = resample(data, field, flagger, "10min", np.mean, max_invalid_total_d=2, max_invalid_consec_d=1)
+ flags = initFlagsLike(data)
+ data1, *_ = resample(data, field, flags, "10min", np.mean, max_invalid_total_d=2, max_invalid_consec_d=1)
assert ~np.isnan(data1[field].iloc[0])
assert np.isnan(data1[field].iloc[1])
assert np.isnan(data1[field].iloc[2])
@@ -83,8 +83,8 @@ def test_interpolateGrid(course_5, course_3):
data_grid, characteristics = course_3()
data['grid'] = data_grid.to_df()
# data = dios.DictOfSeries(data)
- flagger = initFlagsLike(data)
- dataInt, *_ = interpolateIndex(data, 'data', flagger, '1h', 'time', grid_field='grid', inter_limit=10)
+ flags = initFlagsLike(data)
+ dataInt, *_ = interpolateIndex(data, 'data', flags, '1h', 'time', grid_field='grid', inter_limit=10)
def test_offsetCorrecture():
@@ -92,7 +92,7 @@ def test_offsetCorrecture():
data.iloc[30:40] = -100
data.iloc[70:80] = 100
data = dios.DictOfSeries(data)
- flagger = initFlagsLike(data)
- data, _ = correctOffset(data, 'dat', flagger, 40, 20, '3d', 1)
+ flags = initFlagsLike(data)
+ data, _ = correctOffset(data, 'dat', flags, 40, 20, '3d', 1)
assert (data == 0).all()[0]
diff --git a/tests/funcs/test_spikes_detection.py b/tests/funcs/test_spikes_detection.py
index 727ef4157..9481d7eb0 100644
--- a/tests/funcs/test_spikes_detection.py
+++ b/tests/funcs/test_spikes_detection.py
@@ -29,9 +29,9 @@ def spiky_data():
def test_flagMad(spiky_data):
data = spiky_data[0]
field, *_ = data.columns
- flagger = initFlagsLike(data)
- data, flagger_result = flagMAD(data, field, flagger, "1H", flag=BAD)
- flag_result = flagger_result[field]
+ flags = initFlagsLike(data)
+ data, flags_result = flagMAD(data, field, flags, "1H", flag=BAD)
+ flag_result = flags_result[field]
test_sum = (flag_result[spiky_data[1]] == BAD).sum()
assert test_sum == len(spiky_data[1])
@@ -39,9 +39,9 @@ def test_flagMad(spiky_data):
def test_flagSpikesBasic(spiky_data):
data = spiky_data[0]
field, *_ = data.columns
- flagger = initFlagsLike(data)
- data, flagger_result = flagOffset(data, field, flagger, thresh=60, tolerance=10, window="20min", flag=BAD)
- flag_result = flagger_result[field]
+ flags = initFlagsLike(data)
+ data, flags_result = flagOffset(data, field, flags, thresh=60, tolerance=10, window="20min", flag=BAD)
+ flag_result = flags_result[field]
test_sum = (flag_result[spiky_data[1]] == BAD).sum()
assert test_sum == len(spiky_data[1])
@@ -59,14 +59,14 @@ def test_flagSpikesBasic(spiky_data):
def test_flagSpikesLimitRaise(dat):
data, characteristics = dat()
field, *_ = data.columns
- flagger = initFlagsLike(data)
- _, flagger_result = flagRaise(
- data, field, flagger,
+ flags = initFlagsLike(data)
+ _, flags_result = flagRaise(
+ data, field, flags,
thresh=2, intended_freq="10min", raise_window="20min", numba_boost=False, flag=BAD
)
- assert np.all(flagger_result[field][characteristics["raise"]] > UNFLAGGED)
- assert not np.any(flagger_result[field][characteristics["return"]] > UNFLAGGED)
- assert not np.any(flagger_result[field][characteristics["drop"]] > UNFLAGGED)
+ assert np.all(flags_result[field][characteristics["raise"]] > UNFLAGGED)
+ assert not np.any(flags_result[field][characteristics["return"]] > UNFLAGGED)
+ assert not np.any(flags_result[field][characteristics["drop"]] > UNFLAGGED)
# see test/functs/fixtures.py for the 'course_N'
@@ -80,12 +80,12 @@ def test_flagMultivarScores(dat):
s1 = pd.Series(data=s1.values, index=s1.index)
s2 = pd.Series(data=s2.values, index=s1.index)
data = dios.DictOfSeries([s1, s2], columns=["data1", "data2"])
- flagger = initFlagsLike(data)
- _, flagger_result = flagMVScores(
- data, field, flagger, fields=fields, trafo=np.log, iter_start=0.95, n_neighbors=10, flag=BAD
+ flags = initFlagsLike(data)
+ _, flags_result = flagMVScores(
+ data, field, flags, fields=fields, trafo=np.log, iter_start=0.95, n_neighbors=10, flag=BAD
)
for field in fields:
- isflagged = flagger_result[field] > UNFLAGGED
+ isflagged = flags_result[field] > UNFLAGGED
assert isflagged[characteristics["raise"]].all()
assert not isflagged[characteristics["return"]].any()
assert not isflagged[characteristics["drop"]].any()
@@ -99,7 +99,7 @@ def test_grubbs(dat):
crowd_size=1, crowd_spacing=3,
out_val=-10,
)
- flagger = initFlagsLike(data)
- data, result_flagger = flagByGrubbs(data, "data", flagger, winsz=20, min_periods=15, flag=BAD)
- assert np.all(result_flagger["data"][char_dict["drop"]] > UNFLAGGED)
+ flags = initFlagsLike(data)
+ data, result_flags = flagByGrubbs(data, "data", flags, winsz=20, min_periods=15, flag=BAD)
+ assert np.all(result_flags["data"][char_dict["drop"]] > UNFLAGGED)
diff --git a/tests/fuzzy/init.py b/tests/fuzzy/init.py
index 4096823b5..b08bb65d8 100644
--- a/tests/fuzzy/init.py
+++ b/tests/fuzzy/init.py
@@ -25,7 +25,7 @@ from saqc.constants import *
from saqc.core.register import FUNC_MAP
from saqc.core.lib import SaQCFunction
from saqc.lib.types import FreqString, ColumnName, IntegerWindow
-from saqc.core import initFlagsLike, Flags as Flagger
+from saqc.core import initFlagsLike, Flags
MAX_EXAMPLES = 50
# MAX_EXAMPLES = 100000
@@ -77,15 +77,15 @@ def columnNames(draw):
@composite
-def flaggers(draw, data):
+def flagses(draw, data):
"""
- initialize a flagger and set some flags
+ initialize a flags and set some flags
"""
- flagger = initFlagsLike(data)
+ flags = initFlagsLike(data)
for col, srs in data.items():
loc_st = lists(sampled_from(sorted(srs.index)), unique=True, max_size=len(srs) - 1)
- flagger[draw(loc_st), col] = BAD
- return flagger
+ flags[draw(loc_st), col] = BAD
+ return flags
@composite
@@ -116,11 +116,11 @@ def frequencyStrings(draw, _):
@composite
-def dataFieldFlagger(draw):
+def dataFieldFlags(draw):
data = draw(dioses())
field = draw(sampled_from(sorted(data.columns)))
- flagger = draw(flaggers(data))
- return data, field, flagger
+ flags = draw(flagses(data))
+ return data, field, flags
@composite
@@ -138,7 +138,7 @@ def functionKwargs(draw, func: SaQCFunction):
kwargs = {
"data": data,
"field": field,
- "flagger": draw(flaggers(data))
+ "flags": draw(flagses(data))
}
column_name_strategy = lambda _: sampled_from(sorted(c for c in data.columns if c != field))
@@ -149,7 +149,7 @@ def functionKwargs(draw, func: SaQCFunction):
register_type_strategy(IntegerWindow, interger_window_strategy)
for k, v in get_type_hints(func.func).items():
- if k not in {"data", "field", "flagger", "return"}:
+ if k not in {"data", "field", "flags", "return"}:
value = draw(from_type(v))
# if v is TimestampColumnName:
# value = draw(columnNames())
diff --git a/tests/fuzzy/test_masking.py b/tests/fuzzy/test_masking.py
index 9567ea7f8..be77e9872 100644
--- a/tests/fuzzy/test_masking.py
+++ b/tests/fuzzy/test_masking.py
@@ -8,60 +8,60 @@ import pandas as pd
from hypothesis import given, settings
from saqc.constants import *
-from saqc.core import Flags as Flagger
+from saqc.core import Flags
from saqc.core.register import _maskData, _unmaskData
-from tests.fuzzy.init import dataFieldFlagger, MAX_EXAMPLES
+from tests.fuzzy.init import dataFieldFlags, MAX_EXAMPLES
logging.disable(logging.CRITICAL)
@settings(max_examples=MAX_EXAMPLES, deadline=None)
-@given(data_field_flagger=dataFieldFlagger())
-def test_maskingMasksData(data_field_flagger):
+@given(data_field_flags=dataFieldFlags())
+def test_maskingMasksData(data_field_flags):
"""
test if flagged values are replaced by np.nan
"""
- flagger: Flagger
- data_in, field, flagger = data_field_flagger
- data_masked, _ = _maskData(data_in, flagger, columns=[field], to_mask=BAD)
- assert data_masked.aloc[flagger.toDios() == BAD].isna().all(axis=None)
+ flags: Flags
+ data_in, field, flags = data_field_flags
+ data_masked, _ = _maskData(data_in, flags, columns=[field], to_mask=BAD)
+ assert data_masked.aloc[flags.toDios() == BAD].isna().all(axis=None)
@settings(max_examples=MAX_EXAMPLES, deadline=None)
-@given(data_field_flagger=dataFieldFlagger())
-def test_dataMutationPreventsUnmasking(data_field_flagger):
+@given(data_field_flags=dataFieldFlags())
+def test_dataMutationPreventsUnmasking(data_field_flags):
""" test if (un)masking works as expected on data-changes.
if `data` is mutated after `_maskData`, `_unmaskData` should be a no-op
"""
filler = -9999
- data_in, field, flagger = data_field_flagger
- data_masked, mask = _maskData(data_in, flagger, columns=[field], to_mask=BAD)
+ data_in, field, flags = data_field_flags
+ data_masked, mask = _maskData(data_in, flags, columns=[field], to_mask=BAD)
data_masked[field] = filler
- data_out = _unmaskData(data_in, mask, data_masked, flagger, to_mask=BAD)
+ data_out = _unmaskData(data_in, mask, data_masked, flags, to_mask=BAD)
assert (data_out[field] == filler).all(axis=None)
@settings(max_examples=MAX_EXAMPLES, deadline=None)
-@given(data_field_flagger=dataFieldFlagger())
-def test_flaggerMutationPreventsUnmasking(data_field_flagger):
+@given(data_field_flags=dataFieldFlags())
+def test_flagsMutationPreventsUnmasking(data_field_flags):
""" test if (un)masking works as expected on flagger-changes.
if `flagger` is mutated after `_maskData`, `_unmaskData` should be a no-op
"""
- data_in, field, flagger = data_field_flagger
- data_masked, mask = _maskData(data_in, flagger, columns=[field], to_mask=BAD)
- flagger = flagger[field] = UNFLAGGED
- data_out = _unmaskData(data_in, mask, data_masked, flagger, to_mask=BAD)
- assert (data_out.loc[flagger[field] == BAD, field].isna()).all(axis=None)
+ data_in, field, flags = data_field_flags
+ data_masked, mask = _maskData(data_in, flags, columns=[field], to_mask=BAD)
+ flags = flags[field] = UNFLAGGED
+ data_out = _unmaskData(data_in, mask, data_masked, flags, to_mask=BAD)
+ assert (data_out.loc[flags[field] == BAD, field].isna()).all(axis=None)
@settings(max_examples=MAX_EXAMPLES, deadline=None)
-@given(data_field_flagger=dataFieldFlagger())
-def test_reshapingPreventsUnmasking(data_field_flagger):
+@given(data_field_flags=dataFieldFlags())
+def test_reshapingPreventsUnmasking(data_field_flags):
""" test if (un)masking works as expected on index-changes.
If the index of data (and flags) change in the func, the unmasking,
@@ -70,30 +70,30 @@ def test_reshapingPreventsUnmasking(data_field_flagger):
filler = -1111
- data_in, field, flagger = data_field_flagger
- data_masked, mask = _maskData(data_in, flagger, columns=[field], to_mask=BAD)
+ data_in, field, flags = data_field_flags
+ data_masked, mask = _maskData(data_in, flags, columns=[field], to_mask=BAD)
# mutate indexes of `data` and `flagger`
index = data_masked[field].index.to_series()
index.iloc[-len(data_masked[field])//2:] += pd.Timedelta("7.5Min")
data_masked[field] = pd.Series(data=filler, index=index)
- flagger.drop(field)
- flagger[field] = pd.Series(data=flagger[field].values, index=index)
+ flags.drop(field)
+ flags[field] = pd.Series(data=flags[field].values, index=index)
- data_out = _unmaskData(data_in, mask, data_masked, flagger, to_mask=BAD)
+ data_out = _unmaskData(data_in, mask, data_masked, flags, to_mask=BAD)
assert (data_out[field] == filler).all(axis=None)
@settings(max_examples=MAX_EXAMPLES, deadline=None)
-@given(data_field_flagger=dataFieldFlagger())
-def test_unmaskingInvertsMasking(data_field_flagger):
+@given(data_field_flags=dataFieldFlags())
+def test_unmaskingInvertsMasking(data_field_flags):
"""
unmasking data should invert the masking
"""
- data_in, field, flagger = data_field_flagger
- data_masked, mask = _maskData(data_in, flagger, columns=[field], to_mask=BAD)
- data_out = _unmaskData(data_in, mask, data_masked, flagger, to_mask=BAD)
+ data_in, field, flags = data_field_flags
+ data_masked, mask = _maskData(data_in, flags, columns=[field], to_mask=BAD)
+ data_out = _unmaskData(data_in, mask, data_masked, flags, to_mask=BAD)
assert data_in.to_df().equals(data_out.to_df())
--
GitLab
From c58af748b3bc440407490f0ada4278172ee0708b Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 26 Mar 2021 01:23:59 +0100
Subject: [PATCH 095/180] the word 'Flagger' now is History (except for
plotting, which is old and deaf and probably die soon anyway)
---
saqc/core/core.py | 17 +-
saqc/core/flags.py | 2 +-
saqc/core/register.py | 12 +-
saqc/funcs/breaks.py | 21 +-
saqc/funcs/changepoints.py | 8 +-
saqc/funcs/constants.py | 20 +-
saqc/funcs/curvefit.py | 9 +-
saqc/funcs/drift.py | 75 ++--
saqc/funcs/flagtools.py | 54 +--
saqc/funcs/generic.py | 26 +-
saqc/funcs/interpolation.py | 28 +-
saqc/funcs/outliers.py | 77 ++--
saqc/funcs/pattern.py | 20 +-
saqc/funcs/resampling.py | 70 ++--
saqc/funcs/residues.py | 10 +-
saqc/funcs/rolling.py | 9 +-
saqc/funcs/scores.py | 4 +-
saqc/funcs/tools.py | 46 +--
saqc/funcs/transformation.py | 8 +-
sphinx-doc/make_doc_module.py | 2 +-
tests/common.py | 17 +-
tests/core/test_flagger.py | 756 ----------------------------------
tests/fuzzy/test_masking.py | 36 +-
23 files changed, 282 insertions(+), 1045 deletions(-)
delete mode 100644 tests/core/test_flagger.py
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 8041f6f4f..03cb06577 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -4,7 +4,7 @@ from __future__ import annotations
# TODO:
# - integrate plotting into the api
-# - `data` and `flagger` as arguments to `getResult`
+# - `data` and `flags` as arguments to `getResult`
import logging
import copy as stdcopy
@@ -114,15 +114,16 @@ class SaQC(FuncModules):
self._to_call: List[Tuple[ColumnSelector, APIController, SaQCFunction]] = []
def _initFlags(self, data, flags: Union[Flags, None]):
- """ Init the internal flagger object.
+ """ Init the internal Flags-object.
- Ensures that all data columns are present and user passed flags from
- a flags frame or an already initialised flagger are used.
+ Ensures that all data columns are present and user passed
+ flags from a frame or an already initialised Flags-object
+ are used.
"""
if flags is None:
return initFlagsLike(data)
- # add columns that are present in data but not in flagger
+ # add columns that are present in data but not in flags
for c in data.columns.difference(flags.columns):
flags[c] = pd.Series(UNFLAGGED, index=data[c].index, dtype=float)
@@ -202,7 +203,7 @@ class SaQC(FuncModules):
plotAllHook(data, flags)
# This is way faster for big datasets, than to throw everything in the constructor.
- # Simply because of _initFlagger -> merge() -> mergeDios() over all columns.
+ # Simply because of _initFlags -> merge() -> mergeDios() over all columns.
new = self._constructSimple()
new._flags, new._data = flags, data
return new
@@ -213,7 +214,7 @@ class SaQC(FuncModules):
Returns
-------
- data, flagger: (DictOfSeries, DictOfSeries)
+ data, flags: (DictOfSeries, DictOfSeries)
"""
realization = self.evaluate()
@@ -270,7 +271,7 @@ class SaQC(FuncModules):
def _saqcCallFunc(locator, controller, function, data, flags):
# NOTE:
# We assure that all columns in data have an equivalent column in flags,
- # we might have more flagger columns though
+ # we might have more flags columns though
assert data.columns.difference(flags.columns).empty
field = locator.field
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index 1d95adf31..d698cf1d0 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -202,7 +202,7 @@ class Flags:
# technically it would be possible to select a field and set
# the entire column to a scalar flag value (float), but it has
# a high potential, that this is not intended by the user.
- # if desired use ``flagger[:, field] = flag``
+ # if desired use ``flags[:, field] = flag``
if not isinstance(value, pd.Series):
raise ValueError("must pass value of type pd.Series")
diff --git a/saqc/core/register.py b/saqc/core/register.py
index b00f353d0..256b3228a 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -131,14 +131,14 @@ def _postCall(result, old_state: CallState) -> FuncReturnT:
Parameters
----------
result : tuple
- the result from the called function, namely: data and flagger
+ the result from the called function, namely: data and flags
old_state : dict
control keywords from `_preCall`
Returns
-------
- data, flagger : dios.DictOfSeries, saqc.flagger.Flagger
+ data, flags : dios.DictOfSeries, saqc.Flags
"""
data, flags = result
flags = _restoreFlags(flags, old_state)
@@ -264,7 +264,7 @@ def _prepareFlags(flags: Flags, masking) -> Flags:
Prepare flags before each call. Always returns a copy.
Currently this only clears the flags, but in future,
- this should be sliced the flagger to the columns, that
+ this should be sliced the flags to the columns, that
the saqc-function needs.
"""
# Either the index or the columns itself changed
@@ -286,9 +286,9 @@ def _restoreFlags(flags: Flags, old_state: CallState):
out = old_state.flags.copy()
for c in columns:
- # this implicitly squash the new-flagger history (RHS) to a single column, which than is appended to
- # the old history (LHS). The new-flagger history possibly consist of multiple columns, one for each
- # time flags was set to the flagger.
+ # this implicitly squash the new flags history (RHS) to a single column, which than is appended to
+ # the old history (LHS). The new flags history possibly consist of multiple columns, one for each
+ # time a series or scalar was passed to the flags.
out[c] = flags[c]
return out
diff --git a/saqc/funcs/breaks.py b/saqc/funcs/breaks.py
index f3ad9eeb2..b6dd00834 100644
--- a/saqc/funcs/breaks.py
+++ b/saqc/funcs/breaks.py
@@ -41,8 +41,8 @@ def flagMissing(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
nodata : any, default np.nan
A value that defines missing data.
flag : float, default BAD
@@ -52,9 +52,8 @@ def flagMissing(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
"""
datacol = data[field]
if np.isnan(nodata):
@@ -88,8 +87,8 @@ def flagIsolated(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object
+ flags : saqc.Flags
+ A flags object
gap_window : str
The minimum size of the gap before and after a group of valid values, making this group considered an
isolated group. See condition (2) and (3)
@@ -103,8 +102,8 @@ def flagIsolated(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional information related to `data`.
+ flags : saqc.Flags
+ The flags object, holding flags and additional information related to `data`.
Notes
-----
@@ -160,8 +159,8 @@ def flagJumps(
A dictionary of pandas.Series, holding all the data.
field : str
The reference variable, the deviation from wich determines the flagging.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
thresh : float
The threshold, the mean of the values have to change by, to trigger flagging.
winsz : str
diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py
index 83439157e..450a77602 100644
--- a/saqc/funcs/changepoints.py
+++ b/saqc/funcs/changepoints.py
@@ -47,8 +47,8 @@ def flagChangePoints(
A dictionary of pandas.Series, holding all the data.
field : str
The reference variable, the deviation from wich determines the flagging.
- flagger : saqc.flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.flags
+ A flags object, holding flags and additional informations related to `data`.
stat_func : Callable[numpy.array, numpy.array]
A function that assigns a value to every twin window. Left window content will be passed to first variable,
right window content will be passed to the second.
@@ -140,8 +140,8 @@ def assignChangePointCluster(
A dictionary of pandas.Series, holding all the data.
field : str
The reference variable, the deviation from wich determines the flagging.
- flagger : saqc.flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.flags
+ A flags object, holding flags and additional informations related to `data`.
stat_func : Callable[[numpy.array, numpy.array], float]
A function that assigns a value to every twin window. Left window content will be passed to first variable,
right window content will be passed to the second.
diff --git a/saqc/funcs/constants.py b/saqc/funcs/constants.py
index 3791eaabf..6f8d29828 100644
--- a/saqc/funcs/constants.py
+++ b/saqc/funcs/constants.py
@@ -43,8 +43,8 @@ def flagConstants(
A dictionary of pandas.Series, holding all the data.
field : str
Name of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
thresh : float
Upper bound for the maximum total change of an interval to be flagged constant.
window : str
@@ -56,9 +56,9 @@ def flagConstants(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional informations related to `data`.
- Flags values may have changed, relatively to the flagger input.
+ flags : saqc.Flags
+ The flags object, holding flags and additional informations related to `data`.
+ Flags values may have changed, relatively to the flags input.
"""
if not isinstance(window, str):
raise TypeError('window must be offset string.')
@@ -104,8 +104,8 @@ def flagByVariance(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
window : str
Only intervals of minimum size "window" have the chance to get flagged as constant intervals
thresh : float
@@ -125,9 +125,9 @@ def flagByVariance(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional informations related to `data`.
- Flags values may have changed, relatively to the flagger input.
+ flags : saqc.Flags
+ The flags object, holding flags and additional informations related to `data`.
+ Flags values may have changed, relatively to the flags input.
"""
dataseries = data[field]
diff --git a/saqc/funcs/curvefit.py b/saqc/funcs/curvefit.py
index 3465e07d1..4b50693c6 100644
--- a/saqc/funcs/curvefit.py
+++ b/saqc/funcs/curvefit.py
@@ -74,8 +74,8 @@ def fitPolynomial(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-modelled.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
winsz : {str, int}
The size of the window you want to use for fitting. If an integer is passed, the size
refers to the number of periods for every fitting window. If an offset string is passed,
@@ -106,9 +106,8 @@ def fitPolynomial(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
"""
# TODO: some (rater large) parts are functional similar to saqc.funcs.rolling.roll
if data[field].empty:
diff --git a/saqc/funcs/drift.py b/saqc/funcs/drift.py
index 65f5b043d..f47183c6f 100644
--- a/saqc/funcs/drift.py
+++ b/saqc/funcs/drift.py
@@ -55,8 +55,8 @@ def flagDriftFromNorm(
A dictionary of pandas.Series, holding all the data.
field : str
A dummy parameter.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
fields : str
List of fieldnames in data, determining which variables are to be included into the flagging process.
segment_freq : str
@@ -86,9 +86,9 @@ def flagDriftFromNorm(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the input flagger.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the input flags.
Notes
-----
@@ -166,8 +166,8 @@ def flagDriftFromReference(
A dictionary of pandas.Series, holding all the data.
field : str
The reference variable, the deviation from wich determines the flagging.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
fields : str
List of fieldnames in data, determining wich variables are to be included into the flagging process.
segment_freq : str
@@ -186,9 +186,9 @@ def flagDriftFromReference(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the input flagger.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the input flags.
Notes
-----
@@ -255,8 +255,8 @@ def flagDriftFromScaledNorm(
A dictionary of pandas.Series, holding all the data.
field : str
A dummy parameter.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
fields_scale1 : str
List of fieldnames in data to be included into the flagging process which are scaled according to scaling
scheme 1.
@@ -290,9 +290,9 @@ def flagDriftFromScaledNorm(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the input flagger.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the input flags.
References
----------
@@ -386,8 +386,8 @@ def correctExponentialDrift(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to correct.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
maint_data_field : str
The fieldname of the datacolumn holding the maintenance information.
The maint data is to expected to have following form:
@@ -406,9 +406,8 @@ def correctExponentialDrift(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
"""
# 1: extract fit intervals:
if data[maint_data_field].empty:
@@ -480,8 +479,8 @@ def correctRegimeAnomaly(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to correct.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
cluster_field : str
A string denoting the field in data, holding the cluster label for the data you want to correct.
model : Callable
@@ -502,8 +501,8 @@ def correctRegimeAnomaly(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ The quality flags of data
"""
cluster_ser = data[cluster_field]
unique_successive = pd.unique(cluster_ser.values)
@@ -583,8 +582,8 @@ def correctOffset(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to correct.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
max_mean_jump : float
when searching for changepoints in mean - this is the threshold a mean difference in the
sliding window search must exceed to trigger changepoint detection.
@@ -606,8 +605,8 @@ def correctOffset(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ The quality flags of data
"""
data, flags = copy(data, field, flags, field + '_CPcluster')
data, flags = assignChangePointCluster(
@@ -689,8 +688,8 @@ def flagRegimeAnomaly(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
cluster_field : str
The name of the column in data, holding the cluster labels for the samples in field. (has to be indexed
equal to field)
@@ -711,9 +710,9 @@ def flagRegimeAnomaly(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional informations related to `data`.
- Flags values may have changed, relatively to the flagger input.
+ flags : saqc.Flags
+ The flags object, holding flags and additional informations related to `data`.
+ Flags values may have changed, relatively to the flags input.
"""
return assignRegimeAnomaly(
data, field, flags,
@@ -767,8 +766,8 @@ def assignRegimeAnomaly(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
cluster_field : str
The name of the column in data, holding the cluster labels for the samples in field. (has to be indexed
equal to field)
@@ -794,9 +793,9 @@ def assignRegimeAnomaly(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional informations related to `data`.
- Flags values may have changed, relatively to the flagger input.
+ flags : saqc.Flags
+ The flags object, holding flags and additional informations related to `data`.
+ Flags values may have changed, relatively to the flags input.
"""
series = data[cluster_field]
cluster = np.unique(series)
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index 94b04da10..5cb907781 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -24,8 +24,8 @@ def forceFlags(
data container
field : str
columns name that holds the data
- flagger : Flagger
- flagger object
+ flags : saqc.Flags
+ flags object
flag : float, default BAD
flag to set
kwargs : dict
@@ -34,7 +34,7 @@ def forceFlags(
Returns
-------
data : DictOfSeries
- flagger : Flagger
+ flags : saqc.Flags
See Also
--------
@@ -57,15 +57,15 @@ def clearFlags(data: DictOfSeries, field: ColumnName, flags: Flags, **kwargs) ->
data container
field : str
columns name that holds the data
- flagger : Flagger
- flagger object
+ flags : saqc.Flags
+ flags object
kwargs : dict
unused
Returns
-------
data : DictOfSeries
- flagger : Flagger
+ flags : saqc.Flags
See Also
--------
@@ -93,8 +93,8 @@ def flagUnflagged(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
flag : float, default BAD
flag value to set
kwargs : Dict
@@ -104,8 +104,8 @@ def flagUnflagged(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ The quality flags of data
See Also
--------
@@ -128,15 +128,15 @@ def flagGood(data: DictOfSeries, field: ColumnName, flags: Flags, flag=BAD, **kw
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
Returns
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ The quality flags of data
"""
warnings.warn("'flagGood' is deprecated and does nothing, use 'flagUnflagged' instead", DeprecationWarning)
return data, flags
@@ -165,8 +165,8 @@ def flagManual(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
mdata : {pd.Series, pd.Dataframe, DictOfSeries}
The "manually generated" data
mflag : scalar
@@ -189,7 +189,7 @@ def flagManual(
Returns
-------
data : original data
- flagger : modified flagger
+ flags : modified flags
Examples
--------
@@ -204,7 +204,7 @@ def flagManual(
On *dayly* data, with the 'ontime' method, only the provided timestamnps are used.
Bear in mind that only exact timestamps apply, any offset will result in ignoring
the timestamp.
- >>> _, fl = flagManual(data, field, flagger, mdata, mflag=1, method='ontime')
+ >>> _, fl = flagManual(data, field, flags, mdata, mflag=1, method='ontime')
>>> fl[field] > UNFLAGGED
2000-01-31 False
2000-02-01 True
@@ -217,7 +217,7 @@ def flagManual(
Freq: D, dtype: bool
With the 'right-open' method, the mdata is forward fill:
- >>> _, fl = flagManual(data, field, flagger, mdata, mflag=1, method='right-open')
+ >>> _, fl = flagManual(data, field, flags, mdata, mflag=1, method='right-open')
>>> fl[field] > UNFLAGGED
2000-01-31 False
2000-02-01 True
@@ -229,7 +229,7 @@ def flagManual(
Freq: D, dtype: bool
With the 'left-open' method, backward filling is used:
- >>> _, fl = flagManual(data, field, flagger, mdata, mflag=1, method='left-open')
+ >>> _, fl = flagManual(data, field, flags, mdata, mflag=1, method='left-open')
>>> fl[field] > UNFLAGGED
2000-01-31 False
2000-02-01 False
@@ -287,7 +287,7 @@ def flagManual(
@register(masking='none', module="flagtools")
def flagDummy(data: DictOfSeries, field: ColumnName, flags: Flags, **kwargs) -> Tuple[DictOfSeries, Flags]:
"""
- Function does nothing but returning data and flagger.
+ Function does nothing but returning data and flags.
Parameters
----------
@@ -295,15 +295,15 @@ def flagDummy(data: DictOfSeries, field: ColumnName, flags: Flags, **kwargs) ->
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
Returns
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ The quality flags of data
"""
return data, flags
@@ -319,8 +319,8 @@ def flagForceFail(data: DictOfSeries, field: ColumnName, flags: Flags, **kwargs)
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
Raises
------
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index 329514fcf..93a7eec9b 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -108,8 +108,8 @@ def process(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, where you want the result from the generic expressions processing to be written to.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
func : Callable
The data processing function with parameter names that will be
interpreted as data column entries.
@@ -122,9 +122,9 @@ def process(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
The shape of the data may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- The flags shape may have changed relatively to the input flagger.
+ flags : saqc.Flags
+ The quality flags of data
+ The flags shape may have changed relatively to the input flags.
Examples
--------
@@ -184,8 +184,8 @@ def flag(
field : str
The fieldname of the column, where you want the result from the generic expressions evaluation to be projected
to.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
func : Callable
The expression that is to be evaluated is passed in form of a callable, with parameter names that will be
interpreted as data column entries. The Callable must return an boolen array like.
@@ -199,9 +199,9 @@ def flag(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the flags input.
Examples
--------
@@ -247,9 +247,9 @@ def flag(
if field not in flags.columns:
flags[field] = pd.Series(UNFLAGGED, index=mask.index, name=field)
- # if flagger.getFlags(field).empty:
- # flagger = flagger.merge(
- # flagger.initFlags(
+ # if flags.getFlags(field).empty:
+ # flags = flags.merge(
+ # flags.initFlags(
# data=pd.Series(name=field, index=mask.index, dtype=np.float64)))
flags[mask, field] = flag
return data, flags
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index 32fcd2ba4..1c5131257 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -39,8 +39,8 @@ def interpolateByRolling(
field : str
Name of the column, holding the data-to-be-interpolated.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Information related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional Information related to `data`.
winsz : int, str
The size of the window, the aggregation is computed from. An integer define the number of periods to be used,
@@ -65,9 +65,8 @@ def interpolateByRolling(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
"""
data = data.copy()
@@ -118,8 +117,8 @@ def interpolateInvalid(
field : str
Name of the column, holding the data-to-be-interpolated.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Information related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional Information related to `data`.
method : {"linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric",
"polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"}
@@ -146,9 +145,8 @@ def interpolateInvalid(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
"""
inter_data = interpolateNANs(
data[field],
@@ -203,8 +201,8 @@ def interpolateIndex(
field : str
Name of the column, holding the data-to-be-interpolated.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Information related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional Information related to `data`.
freq : str
An Offset String, interpreted as the frequency of
@@ -233,9 +231,9 @@ def interpolateIndex(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values and shape may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values and shape may have changed relatively to the flags input.
"""
if data[field].empty:
return data, flags
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index 844643f61..b16e93d99 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -42,8 +42,8 @@ def flagByStray(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
partition_freq : str, int, or None, default None
Determines the segmentation of the data into partitions, the kNN algorithm is
@@ -146,8 +146,8 @@ def _evalStrayLabels(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the labels to be evaluated.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
fields : list[str]
A list of strings, holding the column names of the variables, the stray labels shall be
projected onto.
@@ -383,8 +383,8 @@ def flagMVScores(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged. (Here a dummy, for structural reasons)
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
fields : List[str]
List of fieldnames, corresponding to the variables that are to be included into the flagging process.
trafo : callable, default lambda x:x
@@ -437,9 +437,9 @@ def flagMVScores(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed, relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed, relatively to the flags input.
Notes
-----
@@ -540,8 +540,8 @@ def flagRaise(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
thresh : float
The threshold, for the total rise (thresh > 0), or total drop (thresh < 0), value courses must
not exceed within a timespan of length `raise_window`.
@@ -568,9 +568,9 @@ def flagRaise(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed, relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed, relatively to the flags input.
Notes
-----
@@ -700,8 +700,8 @@ def flagMAD(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged. (Here a dummy, for structural reasons)
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
window : str
Offset string. Denoting the windows size that the "Z-scored" values have to lie in.
z: float, default 3.5
@@ -713,9 +713,9 @@ def flagMAD(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed, relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed, relatively to the flags input.
References
----------
@@ -780,8 +780,8 @@ def flagOffset(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged. (Here a dummy, for structural reasons)
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
thresh : float
Minimum difference between to values, to consider the latter one as a spike. See condition (1)
tolerance : float
@@ -802,9 +802,9 @@ def flagOffset(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed, relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed, relatively to the flags input.
References
----------
@@ -933,8 +933,8 @@ def flagByGrubbs(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
winsz : {int, str}
The size of the window you want to use for outlier testing. If an integer is passed, the size
refers to the number of periods of every testing window. If a string is passed, it has to be an offset string,
@@ -955,9 +955,9 @@ def flagByGrubbs(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the flags input.
References
----------
@@ -1029,8 +1029,8 @@ def flagRange(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
min : float
Lower bound for valid data.
max : float
@@ -1042,9 +1042,8 @@ def flagRange(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
"""
# using .values is much faster
@@ -1085,8 +1084,8 @@ def flagCrossStatistic(
A dictionary of pandas.Series, holding all the data.
field : str
A dummy parameter.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
fields : str
List of fieldnames in data, determining wich variables are to be included into the flagging process.
thresh : float
@@ -1105,9 +1104,9 @@ def flagCrossStatistic(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the input flagger.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the input flags.
References
----------
diff --git a/saqc/funcs/pattern.py b/saqc/funcs/pattern.py
index 5f4829e9c..564247488 100644
--- a/saqc/funcs/pattern.py
+++ b/saqc/funcs/pattern.py
@@ -39,8 +39,8 @@ def flagPatternByDTW(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to correct.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
ref_field: str
The fieldname in `data' which holds the pattern.
widths: tuple of int
@@ -57,9 +57,9 @@ def flagPatternByDTW(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the flags input.
References
@@ -126,8 +126,8 @@ def flagPatternByWavelet(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to correct.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
ref_field: str
The fieldname in `data` which holds the pattern.
max_distance: float
@@ -142,9 +142,9 @@ def flagPatternByWavelet(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the flags input.
References
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 967966c9c..8bb871cdd 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -75,8 +75,8 @@ def aggregate(
field : str
The fieldname of the column, holding the data-to-be-regularized.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.freq
+ flags : saqc.Flags
+ Container to store quality flags to data. freq
freq : str
The sampling frequency the data is to be aggregated (resampled) at.
@@ -101,9 +101,9 @@ def aggregate(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values and shape may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values and shape may have changed relatively to the flags input.
"""
data, flags = copy(data, field, flags, field + '_original')
@@ -149,8 +149,8 @@ def linear(
field : str
The fieldname of the column, holding the data-to-be-regularized.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.freq
+ flags : saqc.Flags
+ Container to store flags of the data. freq
freq : str
An offset string. The frequency of the grid you want to interpolate your data at.
@@ -160,9 +160,9 @@ def linear(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values and shape may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values and shape may have changed relatively to the flags input.
"""
data, flags = copy(data, field, flags, field + '_original')
@@ -208,8 +208,8 @@ def interpolate(
field : str
The fieldname of the column, holding the data-to-be-regularized.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.freq
+ flags : saqc.Flags
+ Container to store flags of the data. freq
freq : str
An offset string. The frequency of the grid you want to interpolate your data at.
@@ -227,9 +227,9 @@ def interpolate(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values and shape may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values and shape may have changed relatively to the flags input.
"""
data, flags = copy(data, field, flags, field + '_original')
@@ -294,8 +294,8 @@ def mapToOriginal(
field : str
The fieldname of the column, holding the data-to-be-deharmonized.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.freq
+ flags : saqc.Flags
+ Container to store flags of the data. freq
method : {'inverse_fagg', 'inverse_bagg', 'inverse_nagg', 'inverse_fshift', 'inverse_bshift', 'inverse_nshift',
'inverse_interpolation'}
@@ -307,9 +307,9 @@ def mapToOriginal(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values and shape may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values and shape may have changed relatively to the flags input.
"""
newfield = str(field) + '_original'
data, flags = reindexFlags(data, newfield, flags, method, source=field, to_mask=False)
@@ -338,8 +338,8 @@ def shift(
field : str
The fieldname of the column, holding the data-to-be-shifted.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
freq : str
An frequency Offset String that will be interpreted as the sampling rate you want the data to be shifted to.
@@ -366,9 +366,9 @@ def shift(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values and shape may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values and shape may have changed relatively to the flags input.
"""
data, flags = copy(data, field, flags, field + '_original')
return _shift(data, field, flags, freq, method=method, freq_check=freq_check, **kwargs)
@@ -459,8 +459,8 @@ def resample(
field : str
The fieldname of the column, holding the data-to-be-resampled.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
freq : str
An Offset String, that will be interpreted as the frequency you want to resample your data with.
@@ -509,9 +509,9 @@ def resample(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values and shape may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values and shape may have changed relatively to the flags input.
"""
flagged = _isflagged(flags[field], kwargs['to_mask'])
datcol = data[field]
@@ -651,8 +651,8 @@ def reindexFlags(
field : str
The fieldname of the data column, you want to project the source-flags onto.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
method : {'inverse_fagg', 'inverse_bagg', 'inverse_nagg', 'inverse_fshift', 'inverse_bshift', 'inverse_nshift'}
The method used for projection of source flags onto field flags. See description above for more details.
@@ -668,9 +668,9 @@ def reindexFlags(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values and shape may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values and shape may have changed relatively to the flags input.
"""
flagscol = flags[source]
diff --git a/saqc/funcs/residues.py b/saqc/funcs/residues.py
index ad7b88a64..28a62acd1 100644
--- a/saqc/funcs/residues.py
+++ b/saqc/funcs/residues.py
@@ -65,8 +65,8 @@ def calculatePolynomialResidues(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-modelled.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
winsz : {str, int}
The size of the window you want to use for fitting. If an integer is passed, the size
refers to the number of periods for every fitting window. If an offset string is passed,
@@ -95,9 +95,9 @@ def calculatePolynomialResidues(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the flags input.
"""
return fitPolynomial(
diff --git a/saqc/funcs/rolling.py b/saqc/funcs/rolling.py
index 4b8a5f64e..db9b026fa 100644
--- a/saqc/funcs/rolling.py
+++ b/saqc/funcs/rolling.py
@@ -37,8 +37,8 @@ def roll(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-modelled.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
winsz : {int, str}
The size of the window you want to roll with. If an integer is passed, the size
refers to the number of periods for every fitting window. If an offset string is passed,
@@ -66,9 +66,8 @@ def roll(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
"""
data = data.copy()
to_fit = data[field]
diff --git a/saqc/funcs/scores.py b/saqc/funcs/scores.py
index d5b192aa6..02812f44a 100644
--- a/saqc/funcs/scores.py
+++ b/saqc/funcs/scores.py
@@ -57,8 +57,8 @@ def assignKNNScore(
A dictionary of pandas.Series, holding all the data.
field : str
The reference variable, the deviation from wich determines the flagging.
- flagger : saqc.flagger
- A flagger object, holding flags and additional informations related to `data`.fields
+ flags : saqc.flags
+ A flags object, holding flags and additional informations related to `data`.fields
n_neighbors : int, default 10
The number of nearest neighbors to which the distance is comprised in every datapoints scoring calculation.
trafo : Callable[np.array, np.array], default lambda x: x
diff --git a/saqc/funcs/tools.py b/saqc/funcs/tools.py
index 90db87055..f8950debe 100644
--- a/saqc/funcs/tools.py
+++ b/saqc/funcs/tools.py
@@ -23,8 +23,8 @@ def copy(data: DictOfSeries, field: str, flags: Flags, new_field: str, **kwargs)
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to fork (copy).
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
new_field: str
Target name.
@@ -32,10 +32,10 @@ def copy(data: DictOfSeries, field: str, flags: Flags, new_field: str, **kwargs)
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- data shape may have changed relatively to the flagger input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags shape may have changed relatively to the flagger input.
+ data shape may have changed relatively to the flags input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags shape may have changed relatively to the flags input.
"""
if new_field in flags.columns.union(data.columns):
raise ValueError(f"{field}: field already exist")
@@ -49,7 +49,7 @@ def copy(data: DictOfSeries, field: str, flags: Flags, new_field: str, **kwargs)
@register(masking='none', module="tools")
def drop(data: DictOfSeries, field: str, flags: Flags, **kwargs) -> Tuple[DictOfSeries, Flags]:
"""
- The function drops field from the data dios and the flagger.
+ The function drops field from the data dios and the flags.
Parameters
----------
@@ -57,17 +57,17 @@ def drop(data: DictOfSeries, field: str, flags: Flags, **kwargs) -> Tuple[DictOf
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to drop.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
Returns
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- data shape may have changed relatively to the flagger input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags shape may have changed relatively to the flagger input.
+ data shape may have changed relatively to the flags input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags shape may have changed relatively to the flags input.
"""
del data[field]
del flags[field]
@@ -77,7 +77,7 @@ def drop(data: DictOfSeries, field: str, flags: Flags, **kwargs) -> Tuple[DictOf
@register(masking='none', module="tools")
def rename(data: DictOfSeries, field: str, flags: Flags, new_name: str, **kwargs) -> Tuple[DictOfSeries, Flags]:
"""
- The function renames field to new name (in both, the flagger and the data).
+ The function renames field to new name (in both, the flags and the data).
Parameters
----------
@@ -85,8 +85,8 @@ def rename(data: DictOfSeries, field: str, flags: Flags, new_name: str, **kwargs
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to rename.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
new_name : str
String, field is to be replaced with.
@@ -94,8 +94,8 @@ def rename(data: DictOfSeries, field: str, flags: Flags, new_name: str, **kwargs
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ The quality flags of data
"""
data[new_name] = data[field]
flags.history[new_name] = flags.history[field]
@@ -141,8 +141,8 @@ def mask(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-masked.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
mode : {"periodic", "mask_var"}
The masking mode.
- "periodic": parameters "period_start", "period_end" are evaluated to generate a periodical mask
@@ -170,9 +170,9 @@ def mask(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the flags input.
Examples
diff --git a/saqc/funcs/transformation.py b/saqc/funcs/transformation.py
index 48a072909..91952d0f1 100644
--- a/saqc/funcs/transformation.py
+++ b/saqc/funcs/transformation.py
@@ -29,8 +29,8 @@ def transform(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-transformed.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
func : Callable[{pd.Series, np.array}, np.array]
Function to transform data[field] with.
partition_freq : {None, float, str}, default None
@@ -46,8 +46,8 @@ def transform(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ The quality flags of data
"""
data = data.copy()
diff --git a/sphinx-doc/make_doc_module.py b/sphinx-doc/make_doc_module.py
index b2f8ff484..1e508a253 100644
--- a/sphinx-doc/make_doc_module.py
+++ b/sphinx-doc/make_doc_module.py
@@ -83,7 +83,7 @@ def parse_func_dcstrings(m_paths):
continue
dcstr = rm_section(dcstr, 'Returns')
dcstr = rm_parameter(dcstr, 'data')
- dcstr = rm_parameter(dcstr, 'flagger')
+ dcstr = rm_parameter(dcstr, 'flags')
parameters = get_parameter(dcstr)
parameters = [f"{p[0]}={p[1]}" if p[1] else p[0] for p in parameters]
signature = f"def {func.name}({', '.join(parameters)}):"
diff --git a/tests/common.py b/tests/common.py
index 1a3f501a7..21fc6c9c2 100644
--- a/tests/common.py
+++ b/tests/common.py
@@ -11,7 +11,6 @@ from saqc.core import initFlagsLike, Flags
TESTNODATA = (np.nan, -9999)
-TESTFLAGGER = (Flags(),)
def flagAll(data, field, flags, **kwargs):
@@ -47,30 +46,30 @@ def checkDataFlagsInvariants(data, flags, field, identical=True):
Check all invariants that must hold at any point for
* field
* data
- * flagger
+ * flags
* data[field]
- * flagger[field]
+ * flags[field]
* data[field].index
- * flagger[field].index
- * between data and flagger
- * between data[field] and flagger[field]
+ * flags[field].index
+ * between data and flags
+ * between data[field] and flags[field]
Parameters
----------
data : dios.DictOfSeries
data container
- flagger : Flags
+ flags : Flags
flags container
field : str
the field in question
identical : bool, default True
- whether to check indexes of data and flagger to be
+ whether to check indexes of data and flags to be
identical (True, default) of just for equality.
"""
assert isinstance(data, dios.DictOfSeries)
assert isinstance(flags, Flags)
- # all columns in data are in flagger
+ # all columns in data are in flags
assert data.columns.difference(flags.columns).empty
# ------------------------------------------------------------------------
diff --git a/tests/core/test_flagger.py b/tests/core/test_flagger.py
deleted file mode 100644
index 1af9f4710..000000000
--- a/tests/core/test_flagger.py
+++ /dev/null
@@ -1,756 +0,0 @@
-#!/usr/bin/env python
-
-import pytest
-import numpy as np
-import pandas as pd
-from pandas.api.types import is_bool_dtype
-
-import dios
-
-from tests.common import TESTFLAGGER, initData
-
-
-pytestmark = pytest.mark.skip('old flagger tests - rewrite needed')
-
-
-def _getDataset(rows, cols):
- return initData(cols=cols, rows=rows, start_date="2011-01-01", end_date="2011-01-10")
-
-
-DATASETS = [
- _getDataset(0, 1),
- _getDataset(1, 1),
- _getDataset(100, 1),
- # _getDataset(1000, 1),
- _getDataset(0, 4),
- _getDataset(1, 4),
- # _getDataset(100, 4),
- # _getDataset(1000, 4),
- # _getDataset(10000, 40),
- _getDataset(20, 4),
-]
-
-
-def check_all_dios_index_length(tocheck, expected):
- for c in tocheck:
- if len(tocheck[c]) != len(expected[c]):
- return False
- return True
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_initFlags(data, flagger):
- """
- test before:
- - None
- """
-
- newflagger = flagger.initFlags(data)
- assert isinstance(newflagger, type(flagger))
- assert newflagger is not flagger
-
- flags = newflagger.getFlags()
- assert isinstance(flags, dios.DictOfSeries)
-
- assert len(flags.columns) >= len(data.columns)
- assert check_all_dios_index_length(flags, data)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_initFlagsWithFlags(data, flagger):
- flags = dios.DictOfSeries(pd.Series(data=flagger.BAD))
- flagger = flagger.initFlags(flags=flags)
- assert (flagger.flags == flags).all(axis=None)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_getFlags(data, flagger):
- """
- test before:
- - initFlags()
-
- we need to check:
- - access all flags -> get a dios
- - access some columns of flags -> get a dios
- - access one column of flags -> get a series
- """
-
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- # all - dios
- flags0 = flagger.getFlags()
- assert isinstance(flags0, dios.DictOfSeries)
- assert (flags0.columns == data.columns).all()
- assert check_all_dios_index_length(flags0, data)
- for dt in flags0.dtypes:
- assert dt == flagger.dtype
-
- # some - dios
- if len(data.columns) >= 2:
- cols = data.columns[:2].to_list()
- flags1 = flagger.getFlags(cols)
- assert isinstance(flags1, dios.DictOfSeries)
- assert (flags1.columns == data.columns[:2]).all()
- assert check_all_dios_index_length(flags1, data[cols])
- for dt in flags1.dtypes:
- assert dt == flagger.dtype
-
- # series
- flags2 = flagger.getFlags(field)
- assert isinstance(flags2, pd.Series)
- assert flags2.dtype == flagger.dtype
- assert flags2.shape[0] == data[field].shape[0]
- # NOTE: need fix in dios see issue #16 (has very low priority)
- # assert flags2.name in data.columns
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_setFlags(data, flagger):
- """
- test before:
- - initFlags()
- - getFlags()
- """
- flagger = flagger.initFlags(data)
- sl = slice("2011-01-02", "2011-01-05")
- field, *_ = data.columns
-
- base = flagger.getFlags()
-
- flagger_good = flagger.setFlags(field, flag=flagger.GOOD, loc=sl)
- assert isinstance(flagger_good, type(flagger))
- assert flagger_good is not flagger
-
- flags_good = flagger_good.getFlags()
- assert len(flags_good[field]) <= len(base[field])
- assert (flags_good.columns == base.columns).all()
- assert (flags_good.loc[sl, field] == flagger.GOOD).all()
-
- # overflag works BAD > GOOD
- flagger_bad = flagger_good.setFlags(field, flag=flagger.BAD)
- assert (flagger_bad.getFlags(field) == flagger.BAD).all()
-
- # overflag doesn't work GOOD < BAD
- flagger_still_bad = flagger_bad.setFlags(field, flag=flagger.GOOD)
- assert (flagger_still_bad.getFlags(field) == flagger.BAD).all()
-
- # overflag does work with force
- flagger_forced_good = flagger_bad.setFlags(field, flag=flagger.GOOD, force=True)
- assert (flagger_forced_good.getFlags(field) == flagger.GOOD).all()
-
- with pytest.raises(ValueError):
- flagger.setFlags(field=None, flag=flagger.BAD)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_sliceFlagger(data, flagger):
- """
- test before:
- - initFlags()
- - getFlags() inside slice()
- """
- sl = slice(None, None, 3)
-
- flagger = flagger.initFlags(data)
- newflagger = flagger.slice(loc=sl)
- assert isinstance(newflagger, type(flagger))
-
- newflags = newflagger.getFlags()
- assert (newflags.columns == data.columns).all()
- assert check_all_dios_index_length(newflags, data[sl])
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_sliceFlaggerDrop(data, flagger):
- flagger = flagger.initFlags(data)
- with pytest.raises(TypeError):
- flagger.getFlags(field=data.columns, drop="var")
-
- field = data.columns[0]
- expected = data.columns.drop(field)
-
- filtered = flagger.slice(drop=field)
- assert (filtered.getFlags().columns == expected).all(axis=None)
- assert (filtered.getFlags().to_df().index == data[expected].to_df().index).all(axis=None)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_mergeFlagger(data, flagger):
- """
- test before:
- - initFlags()
- - getFlags()
- - setFlags()
- - slice()
- """
- field, *_ = data.columns
- sl = slice(None, None, 3)
-
- this_flagger = flagger.initFlags(data)
- other_flagger = this_flagger.slice(loc=sl).setFlags(field)
- result_flagger = this_flagger.merge(other_flagger)
-
- result_flags = result_flagger.getFlags()
- other_flags = other_flagger.getFlags()
-
- # check flags that was set
- check = result_flags.loc[sl, field] == other_flags[field]
- assert check.all(None)
- # check flags that was not set
- mask = ~result_flags[field].index.isin(other_flags[field].index)
- check = result_flags.loc[mask, field] == result_flagger.UNFLAGGED
- assert check.all(None)
-
- # check unchanged columns
- cols = data.columns.to_list()
- cols.remove(field)
- check = result_flags[cols] == result_flagger.UNFLAGGED
- assert check.all(None)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_mergeFlaggerColumnsDiff(data, flagger):
- """
- test before:
- - initFlags()
- - getFlags()
- - setFlags()
- - slice()
- - merge()
- """
- field, *_ = data.columns
- new_field = field + "_new"
- sl = slice(None, None, 2)
-
- other_data = data.loc[sl]
- other_data.columns = [new_field] + data.columns[1:].to_list()
- other_flagger = flagger.initFlags(other_data)
-
- this_flagger = flagger.initFlags(data).setFlags(field, flag=flagger.BAD)
- result_flagger = this_flagger.merge(other_flagger)
-
- result_flags = result_flagger.getFlags()
- other_flags = other_flagger.getFlags()
-
- # we need to check if
- # - the new column is present
- # - the new column is identical to the original
- # - the other column are unchanged
- # - field-column is BAD
- # - other columns are UNFLAGGED
-
- assert new_field in result_flags
-
- check = result_flags[new_field] == other_flags[new_field]
- assert check.all(None)
-
- check = result_flags[field] == result_flagger.BAD
- assert check.all(None)
-
- cols = data.columns.to_list()
- cols.remove(field)
- check = result_flags[cols] == result_flagger.UNFLAGGED
- assert check.all(None)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_mergeFlaggerIndexDiff(data, flagger):
- """
- test before:
- - initFlags()
- - getFlags()
- - setFlags()
- - slice()
- - merge()
-
- we need to check:
- - index is union of this and other's index
- - indices + values that only in this, should be present
- - indices + values that only in other, should be present
- - indices that in this and other, have values from other
- """
- field, *_ = data.columns
- sl = slice(None, None, 2)
-
- def shiftindex(s):
- s.index = s.index + pd.Timedelta(minutes=2, seconds=25)
- return s
-
- # create a sliced time-shifted version of data
- other_data = data.loc[sl].apply(shiftindex)
- if isinstance(other_data, pd.Series):
- pass
-
- this_flagger = flagger.initFlags(data).setFlags(field, flag=flagger.BAD)
- other_flagger = flagger.initFlags(other_data)
- result_flagger = this_flagger.merge(other_flagger)
-
- result_flags = result_flagger.getFlags()
- this_flags = this_flagger.getFlags()
- other_flags = other_flagger.getFlags()
-
- for c in result_flags:
- t, o, r = this_flags[c], other_flags[c], result_flags[c]
- assert (r.index == t.index.union(o.index)).all()
-
- only_this = t.index.difference(o.index)
- only_other = o.index.difference(t.index)
- both = t.index.intersection(o.index)
-
- # nothing is missing
- assert (r.index == only_this.union(only_other).union(both)).all()
-
- assert (r[only_this] == t[only_this]).all()
- assert (r[only_other] == o[only_other]).all()
- assert (r[both] == o[both]).all()
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_mergeFlaggerOuter(data, flagger):
-
- field = data.columns[0]
-
- data_left = data
- data_right = data.iloc[::2]
-
- left = flagger.initFlags(data=data_left).setFlags(field=field, flag=flagger.BAD)
-
- right = flagger.initFlags(data=data_right).setFlags(field, flag=flagger.GOOD)
-
- merged = left.merge(right, join="outer")
-
- loc = data_right[field].index.difference(data_left[field].index)
- assert (merged.getFlags(field, loc=loc) == flagger.GOOD).all(axis=None)
- assert (merged.getFlags(field, loc=data_left[field].index) == flagger.BAD).all(axis=None)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_mergeFlaggerInner(data, flagger):
-
- field = data.columns[0]
-
- data_left = data
- data_right = data.iloc[::2]
-
- left = flagger.initFlags(data=data_left).setFlags(field=field, flag=flagger.BAD)
-
- right = flagger.initFlags(data=data_right).setFlags(field, flag=flagger.GOOD)
-
- merged = left.merge(right, join="inner")
-
- assert (merged.getFlags(field).index == data_right[field].index).all()
- assert (merged.getFlags(field) == flagger.BAD).all()
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_mergeFlaggerMerge(data, flagger):
-
- field = data.columns[0]
- data_left = data
- data_right = data.iloc[::2]
-
- left = flagger.initFlags(data=data_left).setFlags(field=field, flag=flagger.BAD)
-
- right = flagger.initFlags(data=data_right).setFlags(field, flag=flagger.GOOD)
-
- merged = left.merge(right, join="merge")
-
- loc = data_left[field].index.difference(data_right[field].index)
- assert (merged.getFlags(field, loc=data_right[field].index) == flagger.GOOD).all(axis=None)
- assert (merged.getFlags(field, loc=loc) == flagger.BAD).all(axis=None)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_isFlaggedDios(data, flagger):
- """
- test before:
- - initFlags()
- - setFlags()
- """
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- mask = np.zeros(len(data[field]), dtype=bool)
-
- df_tests = [
- (flagger.isFlagged(), mask),
- (flagger.setFlags(field).isFlagged(), ~mask),
- (flagger.setFlags(field, flag=flagger.GOOD).isFlagged(flag=flagger.GOOD, comparator=">"), mask,),
- (flagger.setFlags(field, flag=flagger.GOOD).isFlagged(flag=flagger.GOOD, comparator="<"), mask,),
- (flagger.setFlags(field, flag=flagger.GOOD).isFlagged(flag=flagger.GOOD, comparator="=="), ~mask,),
- ]
- for flags, expected in df_tests:
- assert np.all(flags[field] == expected)
- assert isinstance(flags, dios.DictOfSeries)
- assert check_all_dios_index_length(flags, data)
- assert (flags.columns == data.columns).all()
- for dt in flags.dtypes:
- assert is_bool_dtype(dt)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_isFlaggedSeries(data, flagger):
- """
- test before:
- - initFlags()
- - setFlags()
- """
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- mask = np.zeros(len(data[field]), dtype=bool)
-
- series_tests = [
- (flagger.isFlagged(field), mask),
- (flagger.setFlags(field).isFlagged(field), ~mask),
- (flagger.setFlags(field, flag=flagger.GOOD).isFlagged(field, flag=flagger.GOOD, comparator=">"), mask,),
- (flagger.setFlags(field, flag=flagger.GOOD).isFlagged(field, flag=flagger.GOOD, comparator="<"), mask,),
- (flagger.setFlags(field, flag=flagger.GOOD).isFlagged(field, flag=flagger.GOOD, comparator="=="), ~mask,),
- ]
- for flags, expected in series_tests:
- assert np.all(flags == expected)
- assert isinstance(flags, pd.Series)
- assert flags.dtype == bool
- assert flags.shape[0] == data[field].shape[0]
- # NOTE: need fix in dios see issue #16 (has very low priority)
- # assert flags.name in data.columns
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_isFlaggedSeries_fail(data, flagger):
- """
- test before:
- - initFlags()
- """
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- fail_tests = [
- {"flag": pd.Series(index=data[field].index, data=flagger.BAD).astype(flagger.dtype)},
- # NOTE: allowed since use of dios
- # {"field": ["var1", "var2"]},
- ]
- for args in fail_tests:
- with pytest.raises(TypeError):
- flagger.isFlagged(**args)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_clearFlags(data, flagger):
- """
- test before:
- - initFlags()
- - getFlags()
- - setFlags()
- - isFlagged()
- """
- flagger = flagger.initFlags(data)
- sl = slice("2011-01-02", "2011-01-05")
- field, *_ = data.columns
-
- base = flagger.getFlags(field)
-
- flagger = flagger.setFlags(field=field, flag=flagger.BAD)
- assert np.sum(flagger.isFlagged(field)) == len(base)
-
- flaggernew = flagger.clearFlags(field)
- assert isinstance(flaggernew, type(flagger))
- assert flaggernew is not flagger
- assert len(flagger.getFlags(field)) == len(data[field])
-
- flagger = flagger.clearFlags(field)
- assert np.sum(flagger.isFlagged(field)) == 0
- assert len(flagger.getFlags(field)) == len(data[field])
-
- flagger = flagger.setFlags(field=field, flag=flagger.BAD)
- assert np.sum(flagger.isFlagged(field)) == len(base)
- assert len(flagger.getFlags(field)) == len(data[field])
-
- flagger = flagger.clearFlags(field, loc=sl)
- assert len(flagger.getFlags(field)) == len(data[field])
- unflagged = flagger.isFlagged(field, loc=sl)
- assert np.sum(unflagged) == 0
- assert np.sum(flagger.isFlagged(field)) == len(data[field]) - len(unflagged)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_dtype(data, flagger):
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- tests = (
- flagger.getFlags(field).astype(str),
- "TEST",
- 55,
- )
-
- for test in tests:
- with pytest.raises(TypeError):
- flagger = flagger.setFlags(field, flag=test)
- assert flagger.getFlags(field).dtype == flagger.dtype
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER[-1:])
-def test_returnCopy(data, flagger):
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- base = flagger.getFlags()
-
- assert flagger.getFlags() is not base
- assert flagger.isFlagged() is not base
- assert flagger.setFlags(field) is not flagger
- assert flagger.clearFlags(field) is not flagger
-
-
-LOC_ILOC_FUNCS = ["isFlagged", "getFlags"]
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-@pytest.mark.parametrize("flaggerfunc", LOC_ILOC_FUNCS)
-def test_loc(data, flagger, flaggerfunc):
- flagger = flagger.initFlags(data)
- sl = slice("2011-01-02", "2011-01-05")
- field, *_ = data.columns
-
- chunk = data.loc[sl, field]
- d = data.loc[sl]
- if d.empty:
- return
-
- m = data[field].index.get_loc(d[field].index[0])
- M = data[field].index.get_loc(d[field].index[-1])
- mask = np.full(len(data[field]), False)
- mask[m:M] = True
-
- flagger_func = getattr(flagger, flaggerfunc)
-
- # masked
- mflags0 = flagger_func(field, loc=mask)
- mflags1 = flagger_func().loc[mask, field]
- mflags2 = flagger_func(field).loc[mask]
- mflags3 = flagger_func(loc=mask)[field]
-
- assert (mflags0 == mflags1).all()
- assert (mflags0 == mflags2).all()
- assert (mflags0 == mflags3).all()
-
- # indexed
- iflags0 = flagger_func(field, loc=chunk.index)
- iflags1 = flagger_func().loc[chunk.index, field]
- iflags2 = flagger_func(field).loc[chunk.index]
- iflags3 = flagger_func(loc=chunk.index)[field]
- assert (iflags0 == iflags1).all()
- assert (iflags0 == iflags2).all()
- assert (iflags0 == iflags3).all()
-
- # sliced
- sflags0 = flagger_func(field, loc=sl)
- sflags1 = flagger_func().loc[sl, field]
- sflags2 = flagger_func(field).loc[sl]
- sflags3 = flagger_func(loc=sl)[field]
- assert (sflags0 == sflags1).all()
- assert (sflags0 == sflags2).all()
- assert (sflags0 == sflags3).all()
-
- assert (sflags0 == iflags0).all()
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_classicUseCases(data, flagger):
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- flagger = flagger.clearFlags(field)
-
- # data-mask, same length than flags
- d = data[field]
- mask = d < (d.max() - d.min()) // 2
- flagged = flagger.setFlags(field, loc=mask, flag=flagger.BAD).isFlagged(field)
- assert (flagged == mask).all()
-
- flagger = flagger.clearFlags(field)
-
- indices = np.arange(0, len(data[field]))
- mask = indices % 3 == 0
- indices = indices[mask]
- # we had some fun with numpy and end up with
- # numpy indices (positional), but with different length..
- # make dt-index with iloc, then pass to loc
- dt_idx = data[field].iloc[indices].index
- flagged = flagger.setFlags(field, loc=dt_idx, flag=flagger.BAD).isFlagged(field)
- assert (flagged.iloc[indices] == flagged[flagged]).all()
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_getFlagsWithExtras(data, flagger):
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- flags, extra = flagger.getFlags(field, full=True)
- assert isinstance(flags, pd.Series)
- assert isinstance(extra, dict)
- for k, v in extra.items():
- assert isinstance(v, pd.Series)
- assert flags.index.equals(v.index)
-
- flags, extra = flagger.getFlags(full=True)
- assert isinstance(flags, dios.DictOfSeries)
- assert isinstance(extra, dict)
- for k, v in extra.items():
- assert isinstance(v, dios.DictOfSeries)
- assert flags.columns.equals(v.columns)
- for c in flags:
- assert flags[c].index.equals(v[c].index)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_replace_delete(data, flagger):
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
- newflagger = flagger.replaceField(field=field, flags=None)
-
- new, newextra = newflagger.getFlags(full=True)
- assert field not in newflagger.flags
- for k in newextra:
- assert field not in newextra[k]
-
- with pytest.raises(ValueError):
- flagger.replaceField(field="i_dont_exist", flags=None)
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_replace_insert(data, flagger):
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
- newfield = 'fooo'
- flags, extra = flagger.getFlags(field, full=True)
- newflagger = flagger.replaceField(field=newfield, flags=flags, **extra)
- old, oldextra = flagger.getFlags(full=True)
- new, newextra = newflagger.getFlags(full=True)
- assert newfield in newflagger.flags
- assert (newflagger._flags[newfield] == flagger._flags[field]).all()
- assert newflagger._flags[newfield] is not flagger._flags[field] # not a copy
- for k in newextra:
- assert newfield in newextra[k]
- assert (newextra[k][newfield] == oldextra[k][field]).all()
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_replace_replace(data, flagger):
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
- flags, extra = flagger.getFlags(field, full=True)
-
- # set everything to DOUBTFUL
- flags[:] = flagger.BAD
- for k, v in extra.items():
- v[:] = flagger.BAD
- extra[k] = v
-
- newflagger = flagger.replaceField(field=field, flags=flags, **extra)
-
- old, oldextra = flagger.getFlags(full=True)
- new, newextra = newflagger.getFlags(full=True)
- assert old.columns.equals(new.columns)
- assert (new[field] == flagger.BAD).all()
-
- assert oldextra.keys() == newextra.keys()
- for k in newextra:
- o, n = oldextra[k], newextra[k]
- assert n.columns.equals(o.columns)
- assert (n[field] == flagger.BAD).all()
-
-
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_flagAfter(flagger):
- idx = pd.date_range("2000", "2001", freq='1M')
- s = pd.Series(0, index=idx)
- data = dios.DictOfSeries(s, columns=['a'])
- exp_base = pd.Series(flagger.UNFLAGGED, index=idx)
-
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- flags = flagger.setFlags(field, loc=s.index[3], flag_after=5).getFlags(field)
- exp = exp_base.copy()
- exp.iloc[3: 3+5+1] = flagger.BAD
- assert (flags == exp).all()
-
- flags = flagger.setFlags(field, loc=s.index[3], flag_after=5, win_flag=flagger.GOOD).getFlags(field)
- exp = exp_base.copy()
- exp.iloc[3: 3+5+1] = flagger.GOOD
- exp[3] = flagger.BAD
- assert (flags == exp).all()
-
- # 3 month < 99 days < 4 month
- flags = flagger.setFlags(field, loc=s.index[3], flag_after="99d").getFlags(field)
- exp = exp_base.copy()
- exp.iloc[3: 3+3+1] = flagger.BAD
- assert (flags == exp).all()
-
- # 3 month < 99 days < 4 month
- flags = flagger.setFlags(field, loc=s.index[3], flag_after="99d", win_flag=flagger.GOOD).getFlags(field)
- exp = exp_base.copy()
- exp.iloc[3: 3+3+1] = flagger.GOOD
- exp[3] = flagger.BAD
- assert (flags == exp).all()
-
-
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_flagBefore(flagger):
- idx = pd.date_range("2000", "2001", freq='1M')
- s = pd.Series(0, index=idx)
- data = dios.DictOfSeries(s, columns=['a'])
- exp_base = pd.Series(flagger.UNFLAGGED, index=idx)
-
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- flags = flagger.setFlags(field, loc=s.index[8], flag_before=5).getFlags(field)
- exp = exp_base.copy()
- exp.iloc[8-5: 8+1] = flagger.BAD
- assert (flags == exp).all()
-
- flags = flagger.setFlags(field, loc=s.index[8], flag_before=5, win_flag=flagger.GOOD).getFlags(field)
- exp = exp_base.copy()
- exp.iloc[8-5: 8+1] = flagger.GOOD
- exp[8] = flagger.BAD
- assert (flags == exp).all()
-
- # 3 month < 99 days < 4 month
- flags = flagger.setFlags(field, loc=s.index[8], flag_before="99d").getFlags(field)
- exp = exp_base.copy()
- exp.iloc[8-3: 8+1] = flagger.BAD
- assert (flags == exp).all()
-
- # 3 month < 99 days < 4 month
- flags = flagger.setFlags(field, loc=s.index[8], flag_before="99d", win_flag=flagger.GOOD).getFlags(field)
- exp = exp_base.copy()
- exp.iloc[8-3: 8+1] = flagger.GOOD
- exp[8] = flagger.BAD
- assert (flags == exp).all()
diff --git a/tests/fuzzy/test_masking.py b/tests/fuzzy/test_masking.py
index be77e9872..cf00f1d98 100644
--- a/tests/fuzzy/test_masking.py
+++ b/tests/fuzzy/test_masking.py
@@ -48,9 +48,9 @@ def test_dataMutationPreventsUnmasking(data_field_flags):
@settings(max_examples=MAX_EXAMPLES, deadline=None)
@given(data_field_flags=dataFieldFlags())
def test_flagsMutationPreventsUnmasking(data_field_flags):
- """ test if (un)masking works as expected on flagger-changes.
+ """ test if (un)masking works as expected on flags-changes.
- if `flagger` is mutated after `_maskData`, `_unmaskData` should be a no-op
+ if `flags` is mutated after `_maskData`, `_unmaskData` should be a no-op
"""
data_in, field, flags = data_field_flags
data_masked, mask = _maskData(data_in, flags, columns=[field], to_mask=BAD)
@@ -73,7 +73,7 @@ def test_reshapingPreventsUnmasking(data_field_flags):
data_in, field, flags = data_field_flags
data_masked, mask = _maskData(data_in, flags, columns=[field], to_mask=BAD)
- # mutate indexes of `data` and `flagger`
+ # mutate indexes of `data` and `flags`
index = data_masked[field].index.to_series()
index.iloc[-len(data_masked[field])//2:] += pd.Timedelta("7.5Min")
data_masked[field] = pd.Series(data=filler, index=index)
@@ -98,8 +98,8 @@ def test_unmaskingInvertsMasking(data_field_flags):
# @settings(max_examples=MAX_EXAMPLES, deadline=None)
-# @given(data_field_flagger=dataFieldFlagger(), func_kwargs=flagFuncsKwargs())
-# def test_maskingPreservesData(data_field_flagger, func_kwargs):
+# @given(data_field_flags=dataFieldFlags(), func_kwargs=flagFuncsKwargs())
+# def test_maskingPreservesData(data_field_flags, func_kwargs):
# """
# no mutations on pre-flagged data
@@ -108,20 +108,20 @@ def test_unmaskingInvertsMasking(data_field_flags):
# are removed
# """
-# data_in, field, flagger = data_field_flagger
+# data_in, field, flags = data_field_flags
-# data_masked, mask = _maskData(data_in, flagger, columns=[field], to_mask=flagger.BAD)
+# data_masked, mask = _maskData(data_in, flags, columns=[field], to_mask=flags.BAD)
# func, kwargs = func_kwargs
-# data_masked, _ = func(data_masked, field, flagger, **kwargs)
-# data_out = _unmaskData(data_in, mask, data_masked, flagger, to_mask=flagger.BAD)
+# data_masked, _ = func(data_masked, field, flags, **kwargs)
+# data_out = _unmaskData(data_in, mask, data_masked, flags, to_mask=flags.BAD)
-# flags_in = flagger.isFlagged(flag=flagger.BAD)
+# flags_in = flags.isFlagged(flag=flags.BAD)
# assert data_in.aloc[flags_in].equals(data_out.aloc[flags_in])
# @settings(max_examples=MAX_EXAMPLES, deadline=None)
-# @given(data_field_flagger=dataFieldFlagger(), func_kwargs=flagFuncsKwargs())
-# def test_maskingEqualsRemoval(data_field_flagger, func_kwargs):
+# @given(data_field_flags=dataFieldFlags(), func_kwargs=flagFuncsKwargs())
+# def test_maskingEqualsRemoval(data_field_flags, func_kwargs):
# """
# calling a function on pre-flagged data should yield the same
# results as calling this function on data where the flagged values
@@ -129,17 +129,17 @@ def test_unmaskingInvertsMasking(data_field_flags):
# """
# func, kwargs = func_kwargs
-# data, field, flagger = data_field_flagger
-# flagged_in = flagger.isFlagged(flag=flagger.BAD, comparator=">=")
+# data, field, flags = data_field_flags
+# flagged_in = flags.isFlagged(flag=flags.BAD, comparator=">=")
# # mask and call
-# data_left, _ = _maskData(data, flagger, columns=[field], to_mask=flagger.BAD)
-# data_left, _ = func(data_left, field, flagger, **kwargs)
+# data_left, _ = _maskData(data, flags, columns=[field], to_mask=flags.BAD)
+# data_left, _ = func(data_left, field, flags, **kwargs)
# # remove and call
# data_right = data.aloc[~flagged_in]
-# flagger_right = flagger.initFlags(flagger.getFlags().aloc[~flagged_in])
-# data_right, _ = func(data_right, field, flagger_right, **kwargs)
+# flags_right = flags.initFlags(flags.getFlags().aloc[~flagged_in])
+# data_right, _ = func(data_right, field, flags_right, **kwargs)
# # NOTE: we need to handle the implicit type conversion in `_maskData`
# data_left_compare = data_left.aloc[~flagged_in]
--
GitLab
From 42bfc8664579c7f643801beb12aee0c85143fef6 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 26 Mar 2021 01:37:14 +0100
Subject: [PATCH 096/180] fixed tiny py3.7 import error
---
saqc/core/history.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/saqc/core/history.py b/saqc/core/history.py
index 8ac0552a7..997192ea9 100644
--- a/saqc/core/history.py
+++ b/saqc/core/history.py
@@ -1,7 +1,8 @@
#!/usr/bin/env python
from __future__ import annotations
-from typing import Tuple, Type, Union, Literal
+from typing import Tuple, Type, Union
+from typing_extensions import Literal
import pandas as pd
import numpy as np
from saqc.constants import *
--
GitLab
From c4416c582588fd36ee15fa0de10e7e3fc95b06b8 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 26 Mar 2021 13:00:13 +0100
Subject: [PATCH 097/180] last fixes, brought pages back online
---
README.md | 9 ++---
compatFlagger_only.patch | 26 -------------
sphinx-doc/Makefile | 2 +-
sphinx-doc/{flagger.rst => flags.rst} | 4 +-
.../getting_started_md/Customizations.md | 26 +++++++------
sphinx-doc/how_to_doc_md/HowToDoc.md | 8 ++--
sphinx-doc/make_doc_module.py | 38 +++++++++----------
sphinx-doc/make_doc_rst.py | 13 ++++---
sphinx-doc/make_html_headings_proppa.py | 17 ++++-----
sphinx-doc/make_md_to_rst.py | 21 +++++-----
10 files changed, 71 insertions(+), 93 deletions(-)
delete mode 100644 compatFlagger_only.patch
rename sphinx-doc/{flagger.rst => flags.rst} (59%)
diff --git a/README.md b/README.md
index 7e79a7340..2e29d3447 100644
--- a/README.md
+++ b/README.md
@@ -81,17 +81,16 @@ The following snippet implements the same configuration given above through
the Python-API:
```python
-import saqc.funcs.outliers
-from saqc import SaQC, SimpleFlagger
+import numpy as np
+from saqc import SaQC
-saqc = saqc = (SaQC(SimpleFlagger(), data)
+saqc = (SaQC(data)
.shiftToFreq("SM2", freq="15Min")
.flagMissing("SM2", nodata=np.nan)
.flagRange("SM(1|2)+", regex=True, min=10, max=60)
.flagMad("SM2", window="30d", z=3.5))
-
-data, flagger = saqc.getResult()
+data, flags = saqc.getResult()
```
## Installation
diff --git a/compatFlagger_only.patch b/compatFlagger_only.patch
deleted file mode 100644
index 1788c31f6..000000000
--- a/compatFlagger_only.patch
+++ /dev/null
@@ -1,26 +0,0 @@
-diff --git a/test/common.py b/test/common.py
-index d5867e94..172d8edd 100644
---- a/test/common.py
-+++ b/test/common.py
-@@ -12,6 +12,7 @@ from saqc.flagger import (
- CategoricalFlagger,
- SimpleFlagger,
- DmpFlagger,
-+ CompatFlagger,
- )
-
-
-@@ -19,9 +20,10 @@ TESTNODATA = (np.nan, -9999)
-
-
- TESTFLAGGER = (
-- CategoricalFlagger(["NIL", "GOOD", "BAD"]),
-- SimpleFlagger(),
-- DmpFlagger(),
-+ # CategoricalFlagger(["NIL", "GOOD", "BAD"]),
-+ # SimpleFlagger(),
-+ # DmpFlagger(),
-+ CompatFlagger(),
- )
-
- def flagAll(data, field, flagger, **kwargs):
diff --git a/sphinx-doc/Makefile b/sphinx-doc/Makefile
index f446fcdc9..992c9c8b9 100644
--- a/sphinx-doc/Makefile
+++ b/sphinx-doc/Makefile
@@ -20,6 +20,7 @@ clean:
rm -f *.automodsumm
rm -f func_modules/*.automodsumm
rm -f intro_modules/*.automodsumm
+ rm -rf ../docs
mkdir _static
# trigger (saqc) customized documentation pipeline
@@ -39,7 +40,6 @@ doc:
python make_html_headings_proppa.py -b "sphinx-doc/_build/html/_api" -p "docs/func_modules" -sr ".."
python make_html_headings_proppa.py -b "sphinx-doc/_build/html/_api" -p "docs/intro_modules" -sr ".."
# clear fake modules/intermediate rest files
-# rm -r ../docs
rm -r getting_started_md_m2r
rm -r how_to_doc_md_m2r
diff --git a/sphinx-doc/flagger.rst b/sphinx-doc/flags.rst
similarity index 59%
rename from sphinx-doc/flagger.rst
rename to sphinx-doc/flags.rst
index d8536aa3e..28e8d605c 100644
--- a/sphinx-doc/flagger.rst
+++ b/sphinx-doc/flags.rst
@@ -1,8 +1,8 @@
-Flagger
+Flags
=======
-.. automodapi:: saqc.flagger
+.. automodapi:: saqc.core.flags
:include-all-objects:
:no-heading:
diff --git a/sphinx-doc/getting_started_md/Customizations.md b/sphinx-doc/getting_started_md/Customizations.md
index 95a9f19b0..b9d9ff359 100644
--- a/sphinx-doc/getting_started_md/Customizations.md
+++ b/sphinx-doc/getting_started_md/Customizations.md
@@ -24,13 +24,17 @@ SaQC provides two ways to integrate custom routines into the system:
In order to make a function usable within the evaluation framework of SaQC the following interface is needed:
```python
+import pandas
+import dios
+import saqc
+
def yourTestFunction(
data: pandas.DataFrame,
field: str,
- flagger: saqc.flagger.BaseFlagger,
- *args: Any,
- **kwargs: Any
- ) -> (dios.DictOfSeries, saqc.flagger.BaseFlagger)
+ flags: saqc.Flags,
+ *args,
+ **kwargs
+ ) -> (dios.DictOfSeries, saqc.Flags)
```
#### Argument Descriptions
@@ -39,21 +43,21 @@ def yourTestFunction(
|-----------|--------------------------------------------------------------------------------------------------|
| `data` | The actual dataset. |
| `field` | The field/column within `data`, that function is processing. |
-| `flagger` | An instance of a flagger, responsible for the translation of test results into quality attributes. |
+| `flags` | An instance of Flags, responsible for the translation of test results into quality attributes. |
| `args` | Any other arguments needed to parameterize the function. |
| `kwargs` | Any other keyword arguments needed to parameterize the function. |
### Integrate into SaQC
-In order make your function available to the system it needs to be registered. We provide the decorator
-[`register`](saqc/functions/register.py) in the module `saqc.functions.register` to integrate your
+In order make your function available to the system it needs to be registered. We provide a decorator
+[`register`](saqc/functions/register.py) with saqc, to integrate your
test functions into SaQC. Here is a complete dummy example:
```python
-from saqc.functions.register import register
+from saqc import register
-@register
-def yourTestFunction(data, field, flagger, *args, **kwargs):
- return data, flagger
+@register()
+def yourTestFunction(data, field, flags, *args, **kwargs):
+ return data, flags
```
### Example
diff --git a/sphinx-doc/how_to_doc_md/HowToDoc.md b/sphinx-doc/how_to_doc_md/HowToDoc.md
index a75361752..9c7d35095 100644
--- a/sphinx-doc/how_to_doc_md/HowToDoc.md
+++ b/sphinx-doc/how_to_doc_md/HowToDoc.md
@@ -109,11 +109,11 @@ But mostly the following sections are sufficient:
^
```
-## Flagger, data, field, etc.
+## Flags, data, field, etc.
use this:
```py
-def foo(data, field, flagger):
+def foo(data, field, flags):
"""
data : dios.DictOfSeries
A saqc-data object.
@@ -121,8 +121,8 @@ def foo(data, field, flagger):
field : str
A field denoting a column in data.
- flagger : saqc.flagger.BaseFlagger
- A saqc-flagger object.
+ flags : saqc.Flags
+ A Flags object.
"""
```
diff --git a/sphinx-doc/make_doc_module.py b/sphinx-doc/make_doc_module.py
index 1e508a253..dd88abb56 100644
--- a/sphinx-doc/make_doc_module.py
+++ b/sphinx-doc/make_doc_module.py
@@ -9,23 +9,25 @@ import pickle
new_line_re = "(\r\n|[\r\n])"
-doc_mod_structure = {'BasicFlagging': ['outliers.flagRange',
- 'breaks.flagMissing'],
- 'BasicFlagging_dcstring': '',
- 'AdvancedFlagging': ['pattern.flagPatternByDTW',
- 'outliers.flagOffset'],
- 'AdvancedFlagging_dcstring': ''}
+doc_mod_structure = {
+ 'BasicFlagging': ['outliers.flagRange', 'breaks.flagMissing'],
+ 'BasicFlagging_dcstring': '',
+ 'AdvancedFlagging': ['pattern.flagPatternByDTW', 'outliers.flagOffset'],
+ 'AdvancedFlagging_dcstring': ''
+}
def rm_section(dcstring, section, _return_section=False):
"""
Detects a section in a docstring and (default) removes it, or (_return_section=True) returns it
"""
- section_re = (f'{new_line_re}(?P<s_name>[^\n\r]{{2,}}){new_line_re}(?P<s_dash>-{{2,}}){new_line_re}')
+ section_re = f'{new_line_re}(?P<s_name>[^\n\r]{{2,}}){new_line_re}(?P<s_dash>-{{2,}}){new_line_re}'
triggers = re.finditer(section_re, dcstring)
- matches = [(trigger.groupdict()['s_name'], trigger.span()) for trigger in triggers if
- len(trigger.groupdict()['s_name']) == len(trigger.groupdict()['s_dash'])] + \
- [(None, (len(dcstring), None))]
+ matches = [
+ (trigger.groupdict()['s_name'], trigger.span())
+ for trigger in triggers
+ if len(trigger.groupdict()['s_name']) == len(trigger.groupdict()['s_dash'])
+ ] + [(None, (len(dcstring), None))]
sections = [m[0] for m in matches]
starts = ends = 0
if section in sections:
@@ -50,7 +52,7 @@ def rm_parameter(dcstring, parameter):
start = re.search(p[0], dcstring).span()[0]
try:
end = dcstring.find(next(paramatches)[0])
- except(StopIteration):
+ except StopIteration:
end = len(re.sub(new_line_re + '$', '', dcstring))
return dcstring[0:start] + dcstring[end:]
@@ -100,7 +102,6 @@ def parse_func_dcstrings(m_paths):
return func_dict
-
def parse_module_dcstrings(m_paths):
mod_dict = {}
for m in m_paths:
@@ -137,20 +138,19 @@ def make_doc_module(targetpath, func_dict, doc_mod_structure):
@click.command()
@click.option(
- "-p", "--pckpath", type=str, required=True, default="saqc/funcs",
+ "-p", "--pckpath", type=str, required=True, default="saqc/funcs",
help="Relative path to the package to be documented (relative to sphinx root)."
)
@click.option(
- "-t", "--targetpath", type=str, required=True, default="docs/intro_modules",
+ "-t", "--targetpath", type=str, required=True, default="docs/intro_modules",
help="Output folder path (relative to sphinx root). Will be overridden if already existent."
)
@click.option(
- "-sr", "--sphinxroot", type=str, required=True, default='..', help="Relative path to the sphinx root."
+ "-sr", "--sphinxroot", type=str, required=True, default='..', help="Relative path to the sphinx root."
)
@click.option(
- "-mo", "--mode", type=str, required=True, default='intro_doc', help="either 'intro_doc' or 'module_doc'."
+ "-mo", "--mode", type=str, required=True, default='intro_doc', help="either 'intro_doc' or 'module_doc'."
)
-
def main(pckpath, targetpath, sphinxroot, mode):
root_path = os.path.abspath(sphinxroot)
pkg_path = os.path.join(root_path, pckpath)
@@ -183,7 +183,7 @@ def main(pckpath, targetpath, sphinxroot, mode):
doc_struct[module + '_dcstring'] = mod_dict[module]
make_doc_module(targetpath, func_dict, doc_struct)
if mode == 'module_doc':
- doc_struct = {m:[] for m in modules}
+ doc_struct = {m: [] for m in modules}
for dm in func_dict.keys():
module = re.search('([^ .]*)\.[^ ]*$', dm).group(1)
doc_struct[module].append(dm)
@@ -191,4 +191,4 @@ def main(pckpath, targetpath, sphinxroot, mode):
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/sphinx-doc/make_doc_rst.py b/sphinx-doc/make_doc_rst.py
index 4271c3241..f8d2a78b2 100644
--- a/sphinx-doc/make_doc_rst.py
+++ b/sphinx-doc/make_doc_rst.py
@@ -4,6 +4,7 @@ import pkgutil
import ast
import shutil
+
def parse_imports(path):
modules = []
file = open(path)
@@ -14,19 +15,19 @@ def parse_imports(path):
file.close()
return modules
+
@click.command()
@click.option(
- "-p", "--pckpath", type=str, required=True, default="saqc/funcs",
+ "-p", "--pckpath", type=str, required=True, default="saqc/funcs",
help="Relative path to the package to be documented (relative to sphinx root)."
)
@click.option(
- "-t", "--targetpath", type=str, required=True, default="sphinx-doc/internal_doc_rst",
+ "-t", "--targetpath", type=str, required=True, default="sphinx-doc/internal_doc_rst",
help="Output folder path (relative to sphinx root). Will be overridden if already existent."
)
@click.option(
- "-sr", "--sphinxroot", type=str, required=True, default='..', help="Relative path to the sphinx root."
+ "-sr", "--sphinxroot", type=str, required=True, default='..', help="Relative path to the sphinx root."
)
-
def main(pckpath, targetpath, sphinxroot):
root_path = os.path.abspath(sphinxroot)
targetpath = os.path.join(root_path, targetpath)
@@ -45,7 +46,7 @@ def main(pckpath, targetpath, sphinxroot):
for module in modules:
imports = parse_imports(os.path.join(pkg_path, f'{module}.py'))
skiplist = [f'\t:skip: {k}' for k in imports]
- section = [module] + ["="*len(module)]
+ section = [module] + ["=" * len(module)]
automodapi_directive = [".. automodapi:: " + pckpath.replace('/', '.') + '.' + module]
no_heading = [f'\t:no-heading:']
to_write = emptyline + section + emptyline + automodapi_directive + skiplist + no_heading
@@ -55,4 +56,4 @@ def main(pckpath, targetpath, sphinxroot):
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/sphinx-doc/make_html_headings_proppa.py b/sphinx-doc/make_html_headings_proppa.py
index 6e4d6b7da..5205666eb 100644
--- a/sphinx-doc/make_html_headings_proppa.py
+++ b/sphinx-doc/make_html_headings_proppa.py
@@ -1,4 +1,3 @@
-
import os
import click
import re
@@ -7,31 +6,29 @@ import pickle
@click.command()
@click.option(
- "-b", "--buildpath", type=str, required=True, default="sphinx-doc/_build/html/_api",
+ "-b", "--buildpath", type=str, required=True, default="sphinx-doc/_build/html/_api",
help="Relative path to the html api files to be manipulated (relative to sphinx root)."
)
@click.option(
- "-sr", "--sphinxroot", type=str, required=True, default='..', help="Relative path to the sphinx root."
+ "-sr", "--sphinxroot", type=str, required=True, default='..', help="Relative path to the sphinx root."
)
@click.option(
- "-p", "--pckpath", type=str, required=True, default="docs/doc_modules/func_modules",
+ "-p", "--pckpath", type=str, required=True, default="docs/doc_modules/func_modules",
help="Relative path to the documented package (relative to sphinx root)."
)
-
def main(buildpath, sphinxroot, pckpath):
root_path = os.path.abspath(sphinxroot)
buildpath = os.path.join(root_path, buildpath)
pckpath = os.path.join(root_path, pckpath)
files = os.listdir(buildpath)
# gather all files from the doc module
- files = [f for f in files if re.search('^docs\.',f)]
+ files = [f for f in files if re.search('^docs\.', f)]
with open(os.path.join(pckpath, 'module_dict.pkl'), 'rb') as file_:
doc_mod_structure = pickle.load(file_)
-
for key in doc_mod_structure.keys():
# search for all function files assigned to the module
- mod_f = [f for f in files if re.search(f'(^|[.]){key}\.[^.]*\.html',f)]
+ mod_f = [f for f in files if re.search(f'(^|[.]){key}\.[^.]*\.html', f)]
for file_ in mod_f:
parts = file_.split('.')
func = parts[-2]
@@ -46,7 +43,7 @@ def main(buildpath, sphinxroot, pckpath):
code = code.replace(old_domain_str, new_domain_str)
with open(os.path.join(buildpath, file_), 'w+') as wf:
wf.write(code)
-
+
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/sphinx-doc/make_md_to_rst.py b/sphinx-doc/make_md_to_rst.py
index 7c1663d76..70ebce512 100644
--- a/sphinx-doc/make_md_to_rst.py
+++ b/sphinx-doc/make_md_to_rst.py
@@ -1,4 +1,3 @@
-
"""
The script generates a folder of rest files from a folder of markdown files.
Markdown Hyperlinks between the files in the folder get converted to rest links so that they function properly in a
@@ -13,6 +12,7 @@ import re
new_line_re = "(\r\n|[\r\n])"
+
def rebaseAbsRoot(path, target, root):
"""
If path and target intersect at root, return relative path from path to target
@@ -34,16 +34,18 @@ def rebaseAbsRoot(path, target, root):
del path[0]
del target[0]
- up_steps = (len(path) - 1)*f'..{os.sep}'
+ up_steps = (len(path) - 1) * f'..{os.sep}'
down_steps = os.sep.join(target)
new_path = os.path.join(up_steps, down_steps)
return new_path
+
def fixTables(f_rst):
body_re = f'((.+){new_line_re})*{new_line_re}((.+){new_line_re})*'
tables = list(re.finditer(f'\.\. list-table::{new_line_re}' + body_re, f_rst))
for t in tables:
tab = t[0]
+
def pic_repl(match):
leading = match.groupdict()['list_level']
pic_dir = match.groupdict()['pic_directive']
@@ -54,6 +56,7 @@ def fixTables(f_rst):
if end_space:
pic_dir = re.sub(f'{new_line_re}[ ]*$', end_space[0], pic_dir)
return pic_dir
+
messy_re = f'(?P<list_level>.*){new_line_re}(?P<pic_directive>[ ]*.. image::[^*-]*)'
# using while loop cause messed pic patterns overlap
tab, repnum = re.subn(messy_re, pic_repl, tab, 1)
@@ -69,14 +72,14 @@ def fixTables(f_rst):
has_content = len([content for content in last_items if re.search('[^\s-]', content)]) > 0
if has_content:
# append empty cells
- tab = tab + (' - \n'*(item_num - last_item_num))
+ tab += ' - \n' * (item_num - last_item_num)
else:
# delete last row (using replace to avoid false meta char interpretation
tab = tab.replace(bullets[-1][0], '')
bullet_num = len(list(re.finditer(f' \*(?P<items>([ ]+-.*{new_line_re})*)', tab)))
if bullet_num == 1:
- #fix empty body table error:
+ # fix empty body table error:
tab = re.sub(':header-rows: [0-9]', ':header-rows: 0', tab)
if tab != t[0]:
@@ -85,7 +88,7 @@ def fixTables(f_rst):
return f_rst
-def fixLinks(f_rst, f ,targetpath):
+def fixLinks(f_rst, f, targetpath):
md_links = list(
re.finditer('(?P<numbered>\. )?`(?P<link_name>[^<`]*) <(?P<md_link>\S*.md)?(#)?(?P<section>[^>]*)?>`_?', f_rst))
for link in md_links:
@@ -94,7 +97,7 @@ def fixLinks(f_rst, f ,targetpath):
if not link_path:
link_path = f
# change directory to point at temporal rest dir (if link isnt relative):
- if os.path.dirname(link_path) is not '':
+ if os.path.dirname(link_path) != '':
link_path = os.path.join(os.path.dirname(link_path) + '_m2r', os.path.basename(link_path))
# rebase the link to relative link if its not
link_path = rebaseAbsRoot(os.path.join(targetpath, f), link_path, 'sphinx-doc')
@@ -123,11 +126,11 @@ def fixLinks(f_rst, f ,targetpath):
@click.command()
@click.option(
- "-p", "--mdpath", type=str, required=True, default="sphinx-doc/getting_started_md",
+ "-p", "--mdpath", type=str, required=True, default="sphinx-doc/getting_started_md",
help="Relative path to the folder containing the .md files to be converted (relative to sphinx root)."
)
@click.option(
- "-sr", "--sphinxroot", type=str, required=True, default='..', help="Relative path to the sphinx root."
+ "-sr", "--sphinxroot", type=str, required=True, default='..', help="Relative path to the sphinx root."
)
def main(mdpath, sphinxroot):
root_path = os.path.abspath(sphinxroot)
@@ -150,4 +153,4 @@ def main(mdpath, sphinxroot):
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
--
GitLab
From e2354e256a77ff142c6d85267464af48a3d8e17e Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Fri, 26 Mar 2021 15:30:12 +0100
Subject: [PATCH 098/180] Flags docu
---
saqc/core/flags.py | 197 ++++++++++++++++++++++++++++++++++++++-----
saqc/core/history.py | 6 +-
2 files changed, 179 insertions(+), 24 deletions(-)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index d698cf1d0..09736a694 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -50,27 +50,116 @@ class _HistAccess:
class Flags:
"""
- flags manipulation
- ------------------
- insert new -> flags['new'] = pd.Series(...)
- set items -> flags['v1'] = pd.Series(...)
- get items -> v0 = flags['v0']
- delete items -> del flags['v0'] / drop('v0')
-
- metadata
- --------
- reading columns -> flags.columns
- renaming column(s) -> flags.columns = pd.Index(['a', 'b', 'c'])
+ Saqc's flags container.
- history
- -------
- get history -> flags.history['v0']
- set history -> flags.history['v0'] = History(...)
+ This container class holds the quality flags associated with the data. It hold key-value pairs, where
+ the key is the name of the column and the value is a ``pandas.Series`` of flags. The index of the series
+ and the key-value pair can be assumed to be immutable, which means, only the *values* of the series can
+ be change, once the series exist.
+ In other words: **an existing column can not be overwritten by a column with a different index.**
- conversion
- ----------
- make a dios -> flags.toDios()
- make a df -> flags.toFrame()
+ The flags can be accessed via ``__getitem__`` and ``__setitem__``, in real life known as the `[]`-operator.
+
+ For the curious:
+ Under the hood, the series are stored in a `history`, which allows the advanced user to retrieve all flags
+ once was set in this object, but in the most cases this is irrelevant. For simplicity one can safely assume,
+ that this class works just stores the flag-series one sets.
+
+ See Also
+ --------
+ initFlagsLike : create a Flags instance, with same dimensions as a reference object.
+ History : class that actually store the flags
+
+ Examples
+ --------
+ We create an empty instance, by calling ``Flags`` without any arguments and then add a column to it.
+
+ >>> from saqc.constants import UNFLAGGED, BAD, DOUBT, UNTOUCHED
+ >>> flags = Flags()
+ >>> flags
+ Empty Flags
+ Columns: []
+ >>> flags['v0'] = pd.Series([BAD,BAD,UNFLAGGED], dtype=float)
+ >>> flags
+ v0 |
+ ======== |
+ 0 255.0 |
+ 1 255.0 |
+ 2 -inf |
+
+ Once the column exist, we cannot overwrite it anymore, with a different series.
+
+ >>> flags['v0'] = pd.Series([666.], dtype=float)
+ Traceback (most recent call last):
+ some file path ...
+ ValueError: Index does not match
+
+ But if we pass a series, which index match it will work,
+ because the series now is interpreted as value-to-set.
+
+ >>> flags['v0'] = pd.Series([DOUBT,UNTOUCHED,DOUBT], dtype=float)
+ >>> flags
+ v0 |
+ ======== |
+ 0 25.0 |
+ 1 255.0 |
+ 2 25.0 |
+
+ As we see above, the column now holds a combination from the values from the
+ first and the second set. This is, because the special constant ``UNTOUCHED``,
+ an alias for ``numpy.nan`` was used. We can inspect all the updates that was
+ made by looking in the history.
+
+ >>> flags.history['v0']
+ 0 1
+ 0 (255.0) 25.0
+ 1 255.0 nan
+ 2 (-inf) 25.0
+
+ As we see now, the second call sets ``25.0`` and shadows (represented by the parentheses) ``(255.0)`` in the
+ first row and ``(-inf)`` in the last, but in the second row ``255.0`` still is valid, because it was
+ `not touched` by the set.
+
+ It is also possible to set values by a mask, which can be interpreted as condidional setting.
+ Imagine we want to `reset` all flags to ``0.`` if the existing flags are lower that ``255.``.
+
+ >>> mask = flags['v0'] < BAD
+ >>> mask
+ 0 True
+ 1 False
+ 2 True
+ dtype: bool
+ >>> flags[mask, 'v0'] = 0
+ >>> flags
+ v0 |
+ ======== |
+ 0 0.0 |
+ 1 255.0 |
+ 2 0.0 |
+
+ The objects you can pass as a row selector (``flags[rows, column]``) are:
+
+ - boolen arraylike, with or without index. Must have same length than the undeliing series.
+ - slices working on the index
+ - ``pd.Index``, which must be a subset of the existing index
+
+ For example, to set `all` values to a scalar value, use a Null-slice:
+
+ >>> flags[:, 'v0'] = 99.0
+ >>> flags
+ v0 |
+ ======= |
+ 0 99.0 |
+ 1 99.0 |
+ 2 99.0 |
+
+ After all calls presented here, the history look like this:
+
+ >>> flags.history['v0']
+ 0 1 2 3
+ 0 (255.0) (25.0) (0.0) 99.0
+ 1 (255.0) (nan) (nan) 99.0
+ 2 (-inf) (25.0) (0.0) 99.0
"""
def __init__(self, raw_data: Optional[Union[DictLike, Flags]] = None, copy: bool = False):
@@ -128,10 +217,26 @@ class Flags:
@property
def columns(self) -> pd.Index:
+ """
+ Column index of the flags container
+
+ Returns
+ -------
+ columns: pd.Index
+ The columns index
+ """
return pd.Index(self._data.keys())
@columns.setter
def columns(self, value: pd.Index):
+ """
+ Set new columns names.
+
+ Parameters
+ ----------
+ value : pd.Index
+ New column names
+ """
if not isinstance(value, pd.Index):
value = pd.Index(value)
@@ -157,6 +262,14 @@ class Flags:
@property
def empty(self) -> bool:
+ """
+ True if flags has no columns.
+
+ Returns
+ -------
+ bool
+ ``True`` if the container has no columns, otherwise ``False``.
+ """
return len(self._data) == 0
def __len__(self) -> int:
@@ -231,8 +344,7 @@ class Flags:
Returns
-------
- Flags
- the same flags object with dropeed column, no copy
+ flags object with dropped column, not a copy
"""
self.__delitem__(key)
@@ -241,12 +353,41 @@ class Flags:
@property
def history(self) -> _HistAccess:
+ """
+ Accessor for the flags history.
+
+ To get a copy of the current history use ``flags.history['var']``.
+ To set a new history use ``flags.history['var'] = value``.
+ The passed value must be a instance of History or must be convertible to a history.
+
+ Returns
+ -------
+ history : History
+ Accessor for the flags history
+
+ See Also
+ --------
+ saqc.core.History : History storage class.
+ """
return _HistAccess(self)
# ----------------------------------------------------------------------
# copy
def copy(self, deep=True):
+ """
+ Copy the flags container.
+
+ Parameters
+ ----------
+ deep : bool, default True
+ If False, a new reference to the Flags container is returned,
+ otherwise the underlying data is also copied.
+
+ Returns
+ -------
+ copy of flags
+ """
return self._constructor(self, copy=deep)
def __copy__(self, deep=True):
@@ -265,6 +406,13 @@ class Flags:
# transformation and representation
def toDios(self) -> dios.DictOfSeries:
+ """
+ Transform the flags container to a ``dios.DictOfSeries``.
+
+ Returns
+ -------
+ dios.DictOfSeries
+ """
di = dios.DictOfSeries(columns=self.columns)
for k, v in self._data.items():
@@ -273,6 +421,13 @@ class Flags:
return di.copy()
def toFrame(self) -> pd.DataFrame:
+ """
+ Transform the flags container to a ``pd.DataFrame``.
+
+ Returns
+ -------
+ pd.DataFrame
+ """
return self.toDios().to_df()
def __repr__(self) -> str:
diff --git a/saqc/core/history.py b/saqc/core/history.py
index 997192ea9..ffb070b72 100644
--- a/saqc/core/history.py
+++ b/saqc/core/history.py
@@ -202,15 +202,15 @@ class History:
Raises
------
- ValueError: on index miss-match or wrong dtype
TypeError: if value is not pd.Series
+ ValueError: on index miss-match or wrong dtype
"""
if isinstance(value, History):
return self._appendHistory(value, force=force)
value = self._validateValue(value)
if len(self) > 0 and not value.index.equals(self.index):
- raise ValueError("Index must be equal to history index")
+ raise ValueError("Index does not match")
self._insert(value, pos=len(self), force=force)
return self
@@ -242,7 +242,7 @@ class History:
"""
self._validateHistWithMask(value.hist, value.mask)
if len(self) > 0 and not value.index.equals(self.index):
- raise ValueError("Index must be equal to history index")
+ raise ValueError("Index does not match")
n = len(self.columns)
value_hist = value.hist
--
GitLab
From b84c17588e6e5ee2576531d53a7a6912faf849c9 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Tue, 30 Mar 2021 22:22:12 +0200
Subject: [PATCH 099/180] test refactorings
---
tests/core/test_translator.py | 38 ++++++++++++++++-------------------
1 file changed, 17 insertions(+), 21 deletions(-)
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index 9152de71b..d881c776e 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -1,6 +1,7 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
+from collections import namedtuple
from dataclasses import dataclass
from typing import Dict, Union, Sequence
@@ -66,25 +67,20 @@ def test_backwardTranslationFail():
translator.backward(flags, None)
-@dataclass
-class _Selector:
- field: str
+def test_dmpTranslator():
-@dataclass
-class _Function:
- name: str
-
+ Selector = namedtuple("Selector", ["field"])
+ Function = namedtuple("Function", ["name"])
-def test_dmpTranslator():
translator = DmpTranslator()
keys = np.array(tuple(translator._backward.keys()) * 50)
flags = _genFlags({"var1": keys, "var2": keys})
flags[:, "var1"] = BAD
to_call = [
- (_Selector("var1"), None, _Function("flagFoo")),
- (_Selector("var1"), None, _Function("flagBar")),
- (_Selector("var2"), None, _Function("flagFoo")),
+ (Selector("var1"), None, Function("flagFoo")),
+ (Selector("var1"), None, Function("flagBar")),
+ (Selector("var2"), None, Function("flagFoo")),
]
tflags = translator.backward(flags, to_call)
assert set(tflags.columns.get_level_values(1)) == {"quality_flag", "quality_comment", "quality_cause"}
@@ -105,15 +101,15 @@ def test_positionalTranslator():
assert (tflags["var1"].iloc[2::3] == "90002").all(axis=None)
-def test_positionalTranslatorIntegration():
- from tests.common import initData
- from saqc import SaQC
+# def test_positionalTranslatorIntegration():
+# from tests.common import initData
+# from saqc import SaQC
- data = initData(3)
- col = data.columns[0]
+# data = initData(3)
+# col = data.columns[0]
- saqc = SaQC(data=data, translator=PositionalTranslator())
- saqc = (saqc
- .breaks.flagMissing(col, flag=2)
- .outliers.flagRange(col, min=3, max=10, flag=2))
- data, flags = saqc.getResult()
+# saqc = SaQC(data=data, translator=PositionalTranslator())
+# saqc = (saqc
+# .breaks.flagMissing(col, flag=2)
+# .outliers.flagRange(col, min=3, max=10, flag=2))
+# data, flags = saqc.getResult()
--
GitLab
From 2f5240148f902f3ce32f55ff4ddf43b179b2c253 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Wed, 31 Mar 2021 09:38:50 +0200
Subject: [PATCH 100/180] enforce a translation mapping for `BAD` an
`UNFLAGGED`
---
saqc/core/translator.py | 18 ++++++++----------
tests/core/test_translator.py | 24 +++++++++++++-----------
2 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
index b381a25ee..4da8865f0 100644
--- a/saqc/core/translator.py
+++ b/saqc/core/translator.py
@@ -35,7 +35,9 @@ class Translator:
def forward(self, flag: UserFlag) -> float:
if flag not in self._forward:
- raise ValueError(f"invalid flag: {flag}")
+ if flag not in self._backward:
+ raise ValueError(f"invalid flag: {flag}")
+ return flag # type: ignore -> if flag is in `self._backward` it is of type float
return self._forward[flag]
def backward(self, flags: Flags, call_stack: CallStack) -> pd.DataFrame:
@@ -65,18 +67,14 @@ class FloatTranslator(Translator):
class DmpTranslator(Translator):
_FORWARD: Dict[str, float] = {"NIL": UNFLAGGED, "OK": GOOD, "DOUBTFUL": DOUBTFUL , "BAD": BAD}
- _BACKWARD: Dict[float, str] = {v: k for k, v in _FORWARD.items()}
+ # _BACKWARD: Dict[float, str] = {v: k for k, v in _FORWARD.items()}
def __init__(self):
- super().__init__(forward=self._FORWARD, backward=self._BACKWARD)
+ super().__init__(forward=self._FORWARD) #, backward=self._BACKWARD)
def _getFieldFunctions(self, field: str, call_stack: CallStack) -> List[str]:
- # NOTE: `SaQC._to_call` shoul probably by an own class prviding such accessors.
- out = []
- for l, _, f in call_stack:
- if l.field == field:
- out.append(f.name)
- return out
+ # NOTE: `SaQC._to_call` should probably by an own class prviding such accessors.
+ return [f.name for l, _, f in call_stack if l.field == field]
def backward(self, flags: Flags, call_stack: CallStack) -> pd.DataFrame:
tflags = super().backward(flags, call_stack)
@@ -100,7 +98,7 @@ class PositionalTranslator(Translator):
_BACKWARD: Dict[float, int] = {UNTOUCHED: 0, UNFLAGGED: 0, GOOD: 0, DOUBTFUL: 1, BAD: 2}
def __init__(self):
- super().__init__(self._FORWARD, self._BACKWARD)
+ super().__init__(forward=self._FORWARD, backward=self._BACKWARD)
def backward(self, flags: Flags, call_stack: CallStack) -> pd.DataFrame:
out = {}
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index d881c776e..f3dcacea2 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -13,10 +13,11 @@ import pytest
from dios import DictOfSeries
from saqc.constants import UNFLAGGED, BAD, DOUBTFUL
-from saqc.core import translator
from saqc.core.translator import PositionalTranslator, Translator, DmpTranslator
from saqc.core.flags import Flags
+from saqc.core.core import SaQC
+from tests.common import initData
def _genTranslators():
for dtype in (str, float, int):
@@ -101,15 +102,16 @@ def test_positionalTranslator():
assert (tflags["var1"].iloc[2::3] == "90002").all(axis=None)
-# def test_positionalTranslatorIntegration():
-# from tests.common import initData
-# from saqc import SaQC
+def test_positionalTranslatorIntegration():
-# data = initData(3)
-# col = data.columns[0]
+ data = initData(3)
+ col = data.columns[0]
-# saqc = SaQC(data=data, translator=PositionalTranslator())
-# saqc = (saqc
-# .breaks.flagMissing(col, flag=2)
-# .outliers.flagRange(col, min=3, max=10, flag=2))
-# data, flags = saqc.getResult()
+ saqc = SaQC(data=data, translator=PositionalTranslator())
+ saqc = (saqc
+ .breaks.flagMissing(col)
+ .outliers.flagRange(col, min=3, max=10))
+ data, flags = saqc.getResult()
+
+ for field in flags.columns:
+ assert flags[field].str.match("9[012]").all()
--
GitLab
From 4b412db7a2ff66a2ce13b26dda44a08540232a61 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Wed, 31 Mar 2021 13:18:32 +0200
Subject: [PATCH 101/180] cleanup: removed unused function call timings
---
saqc/core/core.py | 5 -----
1 file changed, 5 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index f06b44a85..426542709 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -14,7 +14,6 @@ from typing_extensions import Literal
import pandas as pd
import numpy as np
-import timeit
import inspect
from saqc.constants import *
@@ -179,15 +178,11 @@ class SaQC(FuncModules):
for sel, func in self._expandFields(selector, function, data.columns.union(flags.columns)):
logger.debug(f"processing: {sel.field}, {func.name}, {func.keywords}")
- t0 = timeit.default_timer()
try:
data_result, flags_result = _saqcCallFunc(sel, control, func, data, flags)
except Exception as e:
- t1 = timeit.default_timer()
_handleErrors(e, sel.field, control, func, self._error_policy)
continue
- else:
- t1 = timeit.default_timer()
if control.plot:
plotHook(
--
GitLab
From ced8322769dc84ef0aab453ee5f164be38cd38ec Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Wed, 31 Mar 2021 13:19:35 +0200
Subject: [PATCH 102/180] core: added new attribute `_called` to keep track of
what functions were actually called
---
saqc/core/core.py | 24 ++++++++++++++++--------
1 file changed, 16 insertions(+), 8 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 426542709..7629c825f 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -8,7 +8,7 @@ from __future__ import annotations
import logging
import copy as stdcopy
-from typing import List, Tuple, Sequence, Union
+from typing import Tuple, Sequence, Union
from dios import DictOfSeries, to_dios
from typing_extensions import Literal
@@ -24,7 +24,7 @@ from saqc.core.modules import FuncModules
from saqc.funcs.tools import copy
from saqc.lib.plotting import plotHook, plotAllHook
from saqc.core.translator import FloatTranslator, Translator
-from saqc.lib.types import UserFlag
+from saqc.lib.types import UserFlag, CallStack
from saqc.constants import BAD
logger = logging.getLogger("SaQC")
@@ -116,7 +116,9 @@ class SaQC(FuncModules):
translator = FloatTranslator()
self._translator = translator
# NOTE: will be filled by calls to `_wrap`
- self._to_call: List[Tuple[ColumnSelector, APIController, SaQCFunction]] = []
+ self._to_call: CallStack = []
+ # NOTE: will be filled in `evaluate`
+ self._called: CallStack = []
def _initFlags(self, data, flags: Union[Flags, None]):
""" Init the internal Flags-object.
@@ -134,14 +136,19 @@ class SaQC(FuncModules):
return flags
- def _constructSimple(self) -> SaQC:
- return SaQC(
+ def _constructSimple(self, **injectables) -> SaQC:
+ out = SaQC(
data=DictOfSeries(),
flags=Flags(),
nodata=self._nodata,
to_mask=self._to_mask,
error_policy=self._error_policy
)
+ for k, v in injectables.items():
+ if not hasattr(out, k):
+ raise ValueError(f"failed to set unknown attribute: {k}")
+ setattr(out, k, v)
+ return out
def readConfig(self, fname):
from saqc.core.reader import readConfig
@@ -173,13 +180,14 @@ class SaQC(FuncModules):
# NOTE: It would be nicer to separate the plotting into an own
# method instead of intermingling it with the computation
data, flags = self._data, self._flags
-
+ called: CallStack = []
for selector, control, function in self._to_call:
for sel, func in self._expandFields(selector, function, data.columns.union(flags.columns)):
logger.debug(f"processing: {sel.field}, {func.name}, {func.keywords}")
try:
data_result, flags_result = _saqcCallFunc(sel, control, func, data, flags)
+ called.append((sel, control, func))
except Exception as e:
_handleErrors(e, sel.field, control, func, self._error_policy)
continue
@@ -203,7 +211,7 @@ class SaQC(FuncModules):
# This is way faster for big datasets, than to throw everything in the constructor.
# Simply because of _initFlags -> merge() -> mergeDios() over all columns.
- new = self._constructSimple()
+ new = self._constructSimple(_flags=flags, _data=data, _called=called)
new._flags, new._data = flags, data
return new
@@ -222,7 +230,7 @@ class SaQC(FuncModules):
if raw:
return data, flags
- return data.to_df(), self._translator.backward(flags, self._to_call)
+ return data.to_df(), self._translator.backward(flags, realization._called)
def _wrap(self, func: SaQCFunction):
--
GitLab
From ad92b1642be4759b6c4c4846ead52c823c563be3 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Wed, 31 Mar 2021 13:21:11 +0200
Subject: [PATCH 103/180] cleanup: removed trailing whitespaces
---
saqc/core/history.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/saqc/core/history.py b/saqc/core/history.py
index 38c063412..ecc810131 100644
--- a/saqc/core/history.py
+++ b/saqc/core/history.py
@@ -29,7 +29,7 @@ class History:
For more details and a detailed discussion, why this is needed, how this
works and possible other implementations, see #GL143 [1].
-
+
[1] https://git.ufz.de/rdm-software/saqc/-/issues/143
Parameters
@@ -536,4 +536,3 @@ def applyFunctionOnHistory(
new_history.mask = new_history.mask.fillna(False).astype(bool)
return new_history
-
--
GitLab
From 1fb534787a7802a530a6cff4b5328b0893cdd804 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Wed, 31 Mar 2021 13:21:37 +0200
Subject: [PATCH 104/180] new type hints
---
saqc/lib/types.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/saqc/lib/types.py b/saqc/lib/types.py
index f169a5e87..cb5df9186 100644
--- a/saqc/lib/types.py
+++ b/saqc/lib/types.py
@@ -11,14 +11,17 @@ __all__ = [
'IntegerWindow',
'TimestampColumnName',
'CurveFitter',
+ "UserFlag",
+ "CallStack"
]
-from typing import TypeVar, Union, NewType
+from typing import TypeVar, Union, NewType, List, Tuple
from typing_extensions import Protocol, Literal
import numpy as np
import pandas as pd
from dios import DictOfSeries
from saqc.core.flags import Flags
+from saqc.core.lib import SaQCFunction, ColumnSelector, APIController
T = TypeVar("T")
@@ -33,6 +36,8 @@ UserFlag = Union[str, float, int]
# we only support fixed length offsets
FreqString = NewType("FreqString", Literal["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"])
+CallStack = List[Tuple[ColumnSelector, APIController, SaQCFunction]]
+
# we define a bunch of type aliases, mostly needed to generate appropiate fuzzy data through hypothesis
ColumnName = NewType("ColumnName", str)
IntegerWindow = NewType("IntegerWindow", int)
--
GitLab
From 295ec0ba556341197c0e81eae1bbc3ef3689cd9d Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Wed, 31 Mar 2021 13:21:59 +0200
Subject: [PATCH 105/180] translator: added some in code documentation
---
saqc/core/translator.py | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
index 4da8865f0..7f1222137 100644
--- a/saqc/core/translator.py
+++ b/saqc/core/translator.py
@@ -24,6 +24,28 @@ CallStack = List[Tuple[ColumnSelector, APIController, SaQCFunction]]
# to_mask as part of th translator
class Translator:
+ """
+ This class provides the basic translation mechanism and should serve as
+ a base class for every other translation scheme.
+
+ The general translation is realized through dictionary lookups, altough
+ we might need to extend this logic to also allow calls to translation
+ functions in the future. Currently at least one `dict` defining the
+ 'forward' translation from 'user flags' -> 'internal flags' needs to be
+ provided.
+ Optionally a second `dict` can be passed to map 'internal flags' -> 'user flags',
+ if the latter is not given, this 'backwards' translation will inferred as
+ the inverse of the 'forward' translation.
+
+ The translation mechanism imposes a few restrictions:
+ - The scheme must be well definied, i.e. we need a backward translation for
+ every forward translation (each value in `self._forward` needs a key in
+ `self._backward`).
+ - We need translations for the special flags `saqc.constants.UNFLAGGED` and
+ `saqc.constants.BAD`. That implies, that every valid translation scheme
+ provides at least one user flag that maps to `BAD` and one that maps to
+ `UNFLAGGED`.
+ """
def __init__(self, forward: Dict[UserFlag, float], backward: Optional[Dict[float, UserFlag]]=None):
# NOTE: we also add the keys to also allow the usage of internal flags
self._forward = forward
--
GitLab
From 12f1a00053a71ab5751527efb3e7a495656b93b6 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Wed, 31 Mar 2021 13:22:21 +0200
Subject: [PATCH 106/180] translator: working around the additional history
column (GL#182)
---
saqc/core/translator.py | 17 ++++++++--------
tests/core/test_translator.py | 37 ++++++++++++++++++++++++++++-------
2 files changed, 38 insertions(+), 16 deletions(-)
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
index 7f1222137..4c26503c5 100644
--- a/saqc/core/translator.py
+++ b/saqc/core/translator.py
@@ -4,7 +4,7 @@
from __future__ import annotations
import json
-from typing import Dict, List, Tuple, Optional
+from typing import Dict, List, Optional
import numpy as np
import pandas as pd
@@ -12,12 +12,9 @@ import pandas as pd
from dios import DictOfSeries
from saqc.core.flags import Flags, UNTOUCHED, UNFLAGGED, GOOD, DOUBTFUL, BAD
-from saqc.core.lib import APIController, ColumnSelector
-from saqc.core.register import SaQCFunction
from saqc.lib.types import UserFlag
-CallStack = List[Tuple[ColumnSelector, APIController, SaQCFunction]]
# we need: {-np.inf, BAD} as translations
# tanslation schemes müssen gegeben werden, default: IdentityTranslator
@@ -89,7 +86,6 @@ class FloatTranslator(Translator):
class DmpTranslator(Translator):
_FORWARD: Dict[str, float] = {"NIL": UNFLAGGED, "OK": GOOD, "DOUBTFUL": DOUBTFUL , "BAD": BAD}
- # _BACKWARD: Dict[float, str] = {v: k for k, v in _FORWARD.items()}
def __init__(self):
super().__init__(forward=self._FORWARD) #, backward=self._BACKWARD)
@@ -103,7 +99,8 @@ class DmpTranslator(Translator):
out = {}
for field in tflags.columns:
flag_pos = flags.history[field].idxmax()
- flag_funcs = self._getFieldFunctions(field, call_stack)
+ # NOTE: work around the default first column history columns (see GL#182)
+ flag_funcs = ["-"] + self._getFieldFunctions(field, call_stack)
var_flags = {
"quality_flag" : tflags[field],
"quality_comment" : flag_pos.apply(lambda p: json.dumps({"test": flag_funcs[p]})),
@@ -126,7 +123,9 @@ class PositionalTranslator(Translator):
out = {}
for field in flags.columns:
thist = flags.history[field].hist.replace(self._BACKWARD)
- out[field] = (thist
- .astype(int).astype(str)
- .apply(lambda x: "9" + x.sum(), axis="columns"))
+ tflags = (thist
+ .astype(int).astype(str)
+ .apply(lambda x: x.sum(), axis="columns"))
+ # NOTE: work around the default first column history columns (see GL#182)
+ out[field] = "9" + tflags.str.slice(start=1)
return pd.DataFrame(out)
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index f3dcacea2..21c795b5e 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -76,36 +76,59 @@ def test_dmpTranslator():
translator = DmpTranslator()
keys = np.array(tuple(translator._backward.keys()) * 50)
- flags = _genFlags({"var1": keys, "var2": keys})
+ flags = _genFlags({"var1": keys, "var2": keys, "var3": keys})
flags[:, "var1"] = BAD
+ flags[:, "var1"] = DOUBTFUL
+ flags[:, "var2"] = BAD
to_call = [
+ # # NOTE: work around the default first column history columns (see GL#182)
+ # (Selector("var1"), None, Function("-")),
(Selector("var1"), None, Function("flagFoo")),
(Selector("var1"), None, Function("flagBar")),
+ # # NOTE: work around the default first column history columns (see GL#182)
+ # (Selector("var2"), None, Function("-")),
(Selector("var2"), None, Function("flagFoo")),
]
tflags = translator.backward(flags, to_call)
assert set(tflags.columns.get_level_values(1)) == {"quality_flag", "quality_comment", "quality_cause"}
+ assert (tflags.loc[:, ("var1", "quality_flag")] == "DOUBTFUL").all(axis=None)
assert (tflags.loc[:, ("var1", "quality_comment")] == '{"test": "flagBar"}').all(axis=None)
+
+ assert (tflags.loc[:, ("var2", "quality_flag")] == "BAD").all(axis=None)
assert (tflags.loc[:, ("var2", "quality_comment")] == '{"test": "flagFoo"}').all(axis=None)
+ assert (tflags.loc[:, ("var3", "quality_comment")] == '{"test": "-"}').all(axis=None)
+
+
+def test_dmpTranslatorIntegration():
+
+ data = initData(3)
+ col = data.columns[0]
+
+ saqc = SaQC(data=data, translator=DmpTranslator())
+ saqc = (saqc
+ .breaks.flagMissing(col)
+ .outliers.flagRange(col, min=3, max=10))
+ data, flags = saqc.getResult()
+
def test_positionalTranslator():
translator = PositionalTranslator()
- flags = _genFlags({"var1": np.zeros(100), "var2": np.ones(50)})
+ flags = _genFlags({"var1": np.zeros(100), "var2": np.zeros(50)})
flags[1::3, "var1"] = BAD
flags[1::3, "var1"] = DOUBTFUL
flags[2::3, "var1"] = BAD
tflags = translator.backward(flags, None) # type: ignore
- assert (tflags["var2"].dropna() == "91").all(axis=None)
- assert (tflags["var1"].iloc[1::3] == "90210").all(axis=None)
- assert (tflags["var1"].iloc[2::3] == "90002").all(axis=None)
+ assert (tflags["var2"].dropna() == "9").all(axis=None)
+ assert (tflags["var1"].iloc[1::3] == "9210").all(axis=None)
+ assert (tflags["var1"].iloc[2::3] == "9002").all(axis=None)
def test_positionalTranslatorIntegration():
data = initData(3)
- col = data.columns[0]
+ col: str = data.columns[0]
saqc = SaQC(data=data, translator=PositionalTranslator())
saqc = (saqc
@@ -114,4 +137,4 @@ def test_positionalTranslatorIntegration():
data, flags = saqc.getResult()
for field in flags.columns:
- assert flags[field].str.match("9[012]").all()
+ assert flags[field].str.match("^9[012]*$").all()
--
GitLab
From ca04d9c52b24bc47b63720d8b53e04eb1a4c7c96 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Wed, 31 Mar 2021 22:32:34 +0200
Subject: [PATCH 107/180] implement forward translations for flag object
---
saqc/core/core.py | 39 ++++++-----
saqc/core/flags.py | 8 ++-
saqc/core/translator.py | 123 +++++++++++++++++++++++-----------
saqc/lib/types.py | 12 ++--
tests/core/test_translator.py | 69 ++++++++++++-------
5 files changed, 158 insertions(+), 93 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 7629c825f..eaf546b83 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -8,7 +8,7 @@ from __future__ import annotations
import logging
import copy as stdcopy
-from typing import Tuple, Sequence, Union
+from typing import Tuple, Sequence, Union, Optional
from dios import DictOfSeries, to_dios
from typing_extensions import Literal
@@ -24,7 +24,7 @@ from saqc.core.modules import FuncModules
from saqc.funcs.tools import copy
from saqc.lib.plotting import plotHook, plotAllHook
from saqc.core.translator import FloatTranslator, Translator
-from saqc.lib.types import UserFlag, CallStack
+from saqc.lib.types import ExternalFlag, CallStack, CalledStack
from saqc.constants import BAD
logger = logging.getLogger("SaQC")
@@ -112,15 +112,12 @@ class SaQC(FuncModules):
self._to_mask = to_mask
self._flags = self._initFlags(data, flags)
self._error_policy = error_policy
- if translator is None:
- translator = FloatTranslator()
- self._translator = translator
- # NOTE: will be filled by calls to `_wrap`
- self._to_call: CallStack = []
- # NOTE: will be filled in `evaluate`
- self._called: CallStack = []
-
- def _initFlags(self, data, flags: Union[Flags, None]):
+ self._translator = translator or FloatTranslator()
+ self._to_call: CallStack = [] # will be filled by calls to `_wrap`
+ self._called: CalledStack = [] # will be filled in `evaluate`
+
+ @staticmethod
+ def _initFlags(data, flags: Optional[Flags]):
""" Init the internal Flags-object.
Ensures that all data columns are present and user passed
@@ -142,7 +139,8 @@ class SaQC(FuncModules):
flags=Flags(),
nodata=self._nodata,
to_mask=self._to_mask,
- error_policy=self._error_policy
+ error_policy=self._error_policy,
+ translator=self._translator,
)
for k, v in injectables.items():
if not hasattr(out, k):
@@ -156,7 +154,8 @@ class SaQC(FuncModules):
out._to_call.extend(readConfig(fname, self._flags, self._nodata))
return out
- def _expandFields(self, selector: ColumnSelector, func: SaQCFunction, variables: pd.Index) -> Sequence[Tuple[ColumnSelector, SaQCFunction]]:
+ @staticmethod
+ def _expandFields(selector: ColumnSelector, func: SaQCFunction, variables: pd.Index) -> Sequence[Tuple[ColumnSelector, SaQCFunction]]:
if not selector.regex:
return [(selector, func)]
@@ -180,14 +179,14 @@ class SaQC(FuncModules):
# NOTE: It would be nicer to separate the plotting into an own
# method instead of intermingling it with the computation
data, flags = self._data, self._flags
- called: CallStack = []
+ called: CalledStack = []
for selector, control, function in self._to_call:
for sel, func in self._expandFields(selector, function, data.columns.union(flags.columns)):
logger.debug(f"processing: {sel.field}, {func.name}, {func.keywords}")
try:
data_result, flags_result = _saqcCallFunc(sel, control, func, data, flags)
- called.append((sel, control, func))
+ called.append((sel, func))
except Exception as e:
_handleErrors(e, sel.field, control, func, self._error_policy)
continue
@@ -211,9 +210,9 @@ class SaQC(FuncModules):
# This is way faster for big datasets, than to throw everything in the constructor.
# Simply because of _initFlags -> merge() -> mergeDios() over all columns.
- new = self._constructSimple(_flags=flags, _data=data, _called=called)
- new._flags, new._data = flags, data
- return new
+ return self._constructSimple(
+ _flags=flags, _data=data, _called=self._called + called
+ )
def getResult(self, raw=False) -> Union[Tuple[DictOfSeries, Flags], Tuple[pd.DataFrame, pd.DataFrame]]:
"""
@@ -234,7 +233,7 @@ class SaQC(FuncModules):
def _wrap(self, func: SaQCFunction):
- def inner(field: str, *fargs, target: str=None, regex: bool=False, flag: UserFlag=BAD, plot: bool=False, inplace: bool=False, **fkwargs) -> SaQC:
+ def inner(field: str, *fargs, target: str=None, regex: bool=False, flag: ExternalFlag=BAD, plot: bool=False, inplace: bool=False, **fkwargs) -> SaQC:
if self._to_mask is not None:
fkwargs.setdefault('to_mask', self._to_mask)
@@ -251,7 +250,7 @@ class SaQC(FuncModules):
partial = func.bind(
*fargs,
- **{"nodata": self._nodata, "flag": self._translator.forward(flag), **fkwargs}
+ **{"nodata": self._nodata, "flag": self._translator(flag), **fkwargs}
)
out = self if inplace else self.copy(deep=True)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index 7c2f9a231..33f08d7e8 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -200,10 +200,12 @@ class Flags:
if isinstance(item, pd.Series):
item = item.to_frame(name=0)
- elif isinstance(item, History):
+ elif isinstance(item, dios.DictOfSeries):
+ item = item.to_df()
+ elif isinstance(item, (History, pd.DataFrame)):
pass
else:
- raise TypeError(f"cannot init from {type(data.__name__)} of {type(item.__name__)}")
+ raise TypeError(f"cannot init from '{type(data).__name__}' of '{type(item).__name__}'")
result[k] = History(item, copy=copy)
@@ -214,7 +216,7 @@ class Flags:
return type(self)
# ----------------------------------------------------------------------
- # mata data
+ # meta data
@property
def columns(self) -> pd.Index:
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
index 4c26503c5..9544ff021 100644
--- a/saqc/core/translator.py
+++ b/saqc/core/translator.py
@@ -4,7 +4,7 @@
from __future__ import annotations
import json
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Union
import numpy as np
import pandas as pd
@@ -12,14 +12,14 @@ import pandas as pd
from dios import DictOfSeries
from saqc.core.flags import Flags, UNTOUCHED, UNFLAGGED, GOOD, DOUBTFUL, BAD
-from saqc.lib.types import UserFlag
+from saqc.core.history import History
+from saqc.lib.types import ExternalFlag, CalledStack
-
-
-# we need: {-np.inf, BAD} as translations
-# tanslation schemes müssen gegeben werden, default: IdentityTranslator
# to_mask as part of th translator
+ForwardMap = Dict[ExternalFlag, float]
+BackwardMap = Dict[float, ExternalFlag]
+
class Translator:
"""
This class provides the basic translation mechanism and should serve as
@@ -52,63 +52,92 @@ class Translator:
raise ValueError(f"need translations for the special flags `UNFLAGGED` ({UNFLAGGED}) and `BAD` ({BAD})")
self._backward = backward
- def forward(self, flag: UserFlag) -> float:
+ @staticmethod
+ def _translate(flags: Union[Flags, pd.DataFrame], trans_map: Union[ForwardMap, BackwardMap]) -> DictOfSeries:
+ out = DictOfSeries()
+ expected = pd.Index(trans_map.values())
+ for field in flags.columns:
+ out[field] = flags[field].replace(trans_map)
+ diff = pd.Index(out[field]).difference(expected)
+ if not diff.empty:
+ raise ValueError(
+ f"flags were not translated: {diff.drop_duplicates().to_list()}"
+ )
+ return out
+
+ def __call__(self, flag: ExternalFlag) -> float:
if flag not in self._forward:
if flag not in self._backward:
raise ValueError(f"invalid flag: {flag}")
return flag # type: ignore -> if flag is in `self._backward` it is of type float
return self._forward[flag]
- def backward(self, flags: Flags, call_stack: CallStack) -> pd.DataFrame:
+ def forward(self, flags: pd.DataFrame) -> Flags:
+ """
+ translate from 'user flags' to 'internal flags'
+ """
+ return Flags(self._translate(flags, self._forward))
+
+ def backward(self, flags: Flags, call_stack: CalledStack) -> pd.DataFrame:
+ """
+ translate from 'internal flags' to 'user flags', the `call_stack`
+ ensures access to the history of function called on the associated
+ `SaQC`-object.
+ """
# NOTE:
- # - we expect an instance of SaQC as child classes might
- # need to access SaQC._to_call, but maybe the latter is sufficient?
- # - in theory `flags` should only contain valid values,
- # as they all went through `Translator.forward` in practice:
- # who knows...
- out = DictOfSeries()
- expected = pd.Index(self._backward.values())
- for field in flags.columns:
- out[field] = flags[field].replace(self._backward)
- # NOTE: for large datasets (> 100_000 values),
- # dropping first is signifacantly faster
- diff = pd.Index(out[field]).difference(expected)
- if not diff.empty:
- raise ValueError(f"flags were not translated: {expected.to_list()}")
- return out.to_df()
+ return self._translate(flags, self._backward).to_df()
class FloatTranslator(Translator):
+
+ _FORWARD: Dict[float, float] = {-np.inf: -np.inf, **{k: k for k in np.arange(0, 256, dtype=float)}}
+
def __init__(self):
- super().__init__({-np.inf: -np.inf, **{k: k for k in np.arange(0, 256, dtype=float)}})
+ super().__init__(self._FORWARD)
class DmpTranslator(Translator):
- _FORWARD: Dict[str, float] = {"NIL": UNFLAGGED, "OK": GOOD, "DOUBTFUL": DOUBTFUL , "BAD": BAD}
+ _FORWARD: Dict[str, float] = {"NIL": UNFLAGGED, "OK": GOOD, "DOUBTFUL": DOUBTFUL, "BAD": BAD}
+ _COL_LABELS: Dict[str, str] = {"flag": "quality_flag", "comment": "quality_comment", "cause": "quality_cause"}
def __init__(self):
- super().__init__(forward=self._FORWARD) #, backward=self._BACKWARD)
-
- def _getFieldFunctions(self, field: str, call_stack: CallStack) -> List[str]:
- # NOTE: `SaQC._to_call` should probably by an own class prviding such accessors.
- return [f.name for l, _, f in call_stack if l.field == field]
-
- def backward(self, flags: Flags, call_stack: CallStack) -> pd.DataFrame:
+ super().__init__(forward=self._FORWARD)
+
+ @staticmethod
+ def _getFieldFunctions(field: str, call_stack: CalledStack) -> List[str]:
+ """
+ return the names of all functions called on `field`
+
+ NOTE:
+ we prepend an empty string to handle variables that
+ where never tested
+ """
+ return [""] + [f.name for l, f in call_stack if l.field == field]
+
+ def forward(self, flags: pd.DataFrame) -> Flags:
+ cols = flags.columns
+ if not isinstance(cols, pd.MultiIndex):
+ raise TypeError("DMP-Flags need mult-index columns")
+ if set(cols.get_level_values(1)) != set(self._COL_LABELS.values()):
+ raise TypeError(f"DMP-Flags expect the labels 'list(self._COL_LABELS.values)' in the secondary level")
+
+ qflags = flags.xs(key=self._COL_LABELS["flag"], axis="columns", level=1)
+ return super().forward(qflags) # type: ignore
+
+ def backward(self, flags: Flags, call_stack: CalledStack) -> pd.DataFrame:
tflags = super().backward(flags, call_stack)
out = {}
for field in tflags.columns:
flag_pos = flags.history[field].idxmax()
- # NOTE: work around the default first column history columns (see GL#182)
- flag_funcs = ["-"] + self._getFieldFunctions(field, call_stack)
+ flag_funcs = self._getFieldFunctions(field, call_stack)
var_flags = {
- "quality_flag" : tflags[field],
- "quality_comment" : flag_pos.apply(lambda p: json.dumps({"test": flag_funcs[p]})),
- "quality_cause" : "",
+ self._COL_LABELS["flag"] : tflags[field],
+ self._COL_LABELS["comment"] : flag_pos.apply(lambda p: json.dumps({"test": flag_funcs[p]})),
+ self._COL_LABELS["cause"] : "",
}
out[field] = pd.DataFrame(var_flags)
- out = pd.concat(out, axis="columns")
- return out
+ return pd.concat(out, axis="columns")
class PositionalTranslator(Translator):
@@ -119,7 +148,21 @@ class PositionalTranslator(Translator):
def __init__(self):
super().__init__(forward=self._FORWARD, backward=self._BACKWARD)
- def backward(self, flags: Flags, call_stack: CallStack) -> pd.DataFrame:
+ def forward(self, flags: pd.DataFrame) -> Flags:
+ data = {}
+ for field in flags.columns:
+ # drop the first column (i.e. the '9')
+ fflags = pd.DataFrame(
+ flags[field].apply(tuple).tolist(),
+ index=flags[field].index
+ ).iloc[:, 1:]
+
+ tflags = super().forward(fflags.astype(int)).toFrame()
+ tflags.insert(loc=0, column=0, value=pd.Series(UNFLAGGED, index=fflags.index))
+ data[field] = tflags
+ return Flags(data)
+
+ def backward(self, flags: Flags, call_stack: CalledStack) -> pd.DataFrame:
out = {}
for field in flags.columns:
thist = flags.history[field].hist.replace(self._BACKWARD)
diff --git a/saqc/lib/types.py b/saqc/lib/types.py
index cb5df9186..4c8200b3b 100644
--- a/saqc/lib/types.py
+++ b/saqc/lib/types.py
@@ -11,11 +11,12 @@ __all__ = [
'IntegerWindow',
'TimestampColumnName',
'CurveFitter',
- "UserFlag",
- "CallStack"
+ "ExternalFlag",
+ "CallStack",
+ "CalledStack"
]
-from typing import TypeVar, Union, NewType, List, Tuple
+from typing import TypeVar, Union, NewType, List, Tuple, Optional
from typing_extensions import Protocol, Literal
import numpy as np
import pandas as pd
@@ -31,21 +32,20 @@ DiosLikeT = Union[DictOfSeries, pd.DataFrame]
FuncReturnT = [DictOfSeries, Flags]
-UserFlag = Union[str, float, int]
+ExternalFlag = Union[str, float, int]
# we only support fixed length offsets
FreqString = NewType("FreqString", Literal["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"])
CallStack = List[Tuple[ColumnSelector, APIController, SaQCFunction]]
+CalledStack = List[Tuple[ColumnSelector, Optional[SaQCFunction]]]
# we define a bunch of type aliases, mostly needed to generate appropiate fuzzy data through hypothesis
ColumnName = NewType("ColumnName", str)
IntegerWindow = NewType("IntegerWindow", int)
TimestampColumnName = TypeVar("TimestampColumnName", bound=str)
-
# needed for deeper typy hinting magic
class CurveFitter(Protocol):
def __call__(self, data: np.ndarray, *params: float) -> np.ndarray:
...
-
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index 21c795b5e..949c546cb 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -1,8 +1,8 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
+import json
from collections import namedtuple
-from dataclasses import dataclass
from typing import Dict, Union, Sequence
import numpy as np
@@ -19,6 +19,7 @@ from saqc.core.core import SaQC
from tests.common import initData
+
def _genTranslators():
for dtype in (str, float, int):
flags = {dtype(-2): UNFLAGGED, dtype(-1): BAD, **{dtype(f*10): float(f) for f in range(10)}}
@@ -40,12 +41,12 @@ def _genFlags(data: Dict[str, Union[Sequence, pd.Series]]) -> Flags:
def test_forwardTranslation():
for flags, translator in _genTranslators():
for k, expected in flags.items():
- got = translator.forward(k)
+ got = translator(k)
assert expected == got or np.isnan([got, expected]).all()
for k in ["bad", 3.14, max]:
with pytest.raises(ValueError):
- translator.forward(k)
+ translator(k)
def test_backwardTranslation():
@@ -81,35 +82,20 @@ def test_dmpTranslator():
flags[:, "var1"] = DOUBTFUL
flags[:, "var2"] = BAD
to_call = [
- # # NOTE: work around the default first column history columns (see GL#182)
- # (Selector("var1"), None, Function("-")),
- (Selector("var1"), None, Function("flagFoo")),
- (Selector("var1"), None, Function("flagBar")),
- # # NOTE: work around the default first column history columns (see GL#182)
- # (Selector("var2"), None, Function("-")),
- (Selector("var2"), None, Function("flagFoo")),
+ (Selector("var1"), Function("flagFoo")),
+ (Selector("var1"), Function("flagBar")),
+ (Selector("var2"), Function("flagFoo")),
]
tflags = translator.backward(flags, to_call)
assert set(tflags.columns.get_level_values(1)) == {"quality_flag", "quality_comment", "quality_cause"}
+
assert (tflags.loc[:, ("var1", "quality_flag")] == "DOUBTFUL").all(axis=None)
assert (tflags.loc[:, ("var1", "quality_comment")] == '{"test": "flagBar"}').all(axis=None)
assert (tflags.loc[:, ("var2", "quality_flag")] == "BAD").all(axis=None)
assert (tflags.loc[:, ("var2", "quality_comment")] == '{"test": "flagFoo"}').all(axis=None)
- assert (tflags.loc[:, ("var3", "quality_comment")] == '{"test": "-"}').all(axis=None)
-
-
-def test_dmpTranslatorIntegration():
-
- data = initData(3)
- col = data.columns[0]
-
- saqc = SaQC(data=data, translator=DmpTranslator())
- saqc = (saqc
- .breaks.flagMissing(col)
- .outliers.flagRange(col, min=3, max=10))
- data, flags = saqc.getResult()
+ assert (tflags.loc[:, ("var3", "quality_comment")] == '{"test": ""}').all(axis=None)
def test_positionalTranslator():
@@ -130,7 +116,8 @@ def test_positionalTranslatorIntegration():
data = initData(3)
col: str = data.columns[0]
- saqc = SaQC(data=data, translator=PositionalTranslator())
+ translator = PositionalTranslator()
+ saqc = SaQC(data=data, translator=translator)
saqc = (saqc
.breaks.flagMissing(col)
.outliers.flagRange(col, min=3, max=10))
@@ -138,3 +125,37 @@ def test_positionalTranslatorIntegration():
for field in flags.columns:
assert flags[field].str.match("^9[012]*$").all()
+ round_trip = translator.backward(translator.forward(flags), saqc._called)
+
+ assert (flags.values == round_trip.values).all()
+ assert (flags.index == round_trip.index).all()
+ assert (flags.columns == round_trip.columns).all()
+
+
+def test_dmpTranslatorIntegration():
+
+ data = initData(3)
+ col = data.columns[0]
+
+ translator = DmpTranslator()
+ saqc = SaQC(data=data, translator=translator)
+ saqc = (saqc
+ .breaks.flagMissing(col)
+ .outliers.flagRange(col, min=3, max=10))
+ data, flags = saqc.getResult()
+
+ qflags = flags.xs("quality_flag", axis="columns", level=1)
+ qfunc = flags.xs("quality_comment", axis="columns", level=1).applymap(lambda v: json.loads(v)["test"])
+ qcause = flags.xs("quality_cause", axis="columns", level=1)
+
+ assert qflags.isin(translator._forward.keys()).all(axis=None)
+ assert qfunc.isin({"", "breaks.flagMissing", "outliers.flagRange"}).all(axis=None)
+ assert (qcause == "").all(axis=None)
+
+ round_trip = translator.backward(translator.forward(flags), saqc._called)
+ assert round_trip.xs("quality_flag", axis="columns", level=1).equals(qflags)
+ assert (round_trip
+ .xs("quality_comment", axis="columns", level=1)
+ .applymap(lambda v: json.loads(v)["test"] == "")
+ .all(axis=None))
+
--
GitLab
From 83fc5ecddaa7979f67caad6512cfa69cc720c644 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Wed, 31 Mar 2021 22:33:26 +0200
Subject: [PATCH 108/180] move to_mask into the translators
---
saqc/core/core.py | 7 ++-----
saqc/core/register.py | 6 +++---
saqc/core/translator.py | 5 ++++-
3 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index eaf546b83..5369f60ad 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -104,12 +104,11 @@ _setup()
class SaQC(FuncModules):
- def __init__(self, data, flags=None, translator: Translator=None, nodata=np.nan, to_mask=None, error_policy="raise"):
+ def __init__(self, data, flags=None, translator: Translator=None, nodata=np.nan, error_policy="raise"):
super().__init__(self)
data, flags = _prepInput(data, flags)
self._data = data
self._nodata = nodata
- self._to_mask = to_mask
self._flags = self._initFlags(data, flags)
self._error_policy = error_policy
self._translator = translator or FloatTranslator()
@@ -138,7 +137,6 @@ class SaQC(FuncModules):
data=DictOfSeries(),
flags=Flags(),
nodata=self._nodata,
- to_mask=self._to_mask,
error_policy=self._error_policy,
translator=self._translator,
)
@@ -235,8 +233,7 @@ class SaQC(FuncModules):
def inner(field: str, *fargs, target: str=None, regex: bool=False, flag: ExternalFlag=BAD, plot: bool=False, inplace: bool=False, **fkwargs) -> SaQC:
- if self._to_mask is not None:
- fkwargs.setdefault('to_mask', self._to_mask)
+ fkwargs.setdefault('to_mask', self._translator.TO_MASK)
control = APIController(
plot=plot
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 256b3228a..06f0df7aa 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -169,7 +169,7 @@ def _getMaskingColumns(data: dios.DictOfSeries, field: str, masking: MaskingStrT
def _getMaskingThresh(masking, kwargs, fname):
"""
- Check the correct usage of the `to_mask` keyword, iff passed, otherwise return a default.
+ Check the correct usage of the `to_mask` keyword, if passed, otherwise return a default.
Parameters
----------
@@ -179,7 +179,7 @@ def _getMaskingThresh(masking, kwargs, fname):
The kwargs that will be passed to the saqc-function, possibly contain ``to_mask``.
fname : str
The name of the saqc-function to be called later (not here), to use in meaningful
- error messages
+ error messages
Returns
-------
@@ -205,7 +205,7 @@ def _getMaskingThresh(masking, kwargs, fname):
if masking == 'none' and thresh not in (False, np.inf):
# TODO: fix warning reference to docu
- warnings.warn(f"the saqc-function {fname!r} ignore masking and therefore does not evaluate the passed "
+ warnings.warn(f"the saqc-function {fname!r} ignores masking and therefore does not evaluate the passed "
f"'to_mask'-keyword. Please refer to the documentation: TODO")
if thresh is True: # masking ON
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
index 9544ff021..327853d76 100644
--- a/saqc/core/translator.py
+++ b/saqc/core/translator.py
@@ -43,7 +43,10 @@ class Translator:
provides at least one user flag that maps to `BAD` and one that maps to
`UNFLAGGED`.
"""
- def __init__(self, forward: Dict[UserFlag, float], backward: Optional[Dict[float, UserFlag]]=None):
+
+ TO_MASK = True
+
+ def __init__(self, forward: ForwardMap, backward: Optional[BackwardMap]=None):
# NOTE: we also add the keys to also allow the usage of internal flags
self._forward = forward
if backward is None:
--
GitLab
From 829674273e8a1565f9df1f4231a7092c5e2223ef Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 1 Apr 2021 10:17:37 +0200
Subject: [PATCH 109/180] added/fixed type hints
---
saqc/core/core.py | 4 ++--
saqc/core/flags.py | 2 +-
saqc/lib/types.py | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 5369f60ad..70ac3e2e3 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -24,7 +24,7 @@ from saqc.core.modules import FuncModules
from saqc.funcs.tools import copy
from saqc.lib.plotting import plotHook, plotAllHook
from saqc.core.translator import FloatTranslator, Translator
-from saqc.lib.types import ExternalFlag, CallStack, CalledStack
+from saqc.lib.types import ExternalFlag, CallStack, CalledStack, FuncReturnT, PandasLike
from saqc.constants import BAD
logger = logging.getLogger("SaQC")
@@ -50,7 +50,7 @@ def _handleErrors(exc: Exception, field: str, control: APIController, func: SaQC
# TODO: shouldt the code/function go to Saqc.__init__ ?
-def _prepInput(data, flags):
+def _prepInput(data: PandasLike, flags: Optional[Union[DictOfSeries, pd.DataFrame, Flags]]) -> Tuple[DictOfSeries, Optional[Flags]]:
dios_like = (DictOfSeries, pd.DataFrame)
if isinstance(data, pd.Series):
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index 33f08d7e8..1cd7d1481 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -292,7 +292,7 @@ class Flags:
return self._cache[key].copy()
def __setitem__(self, key: SelectT, value: ValueT):
- # force-KW is internal available only
+ # force-KW is only internally available
if isinstance(key, tuple):
if len(key) != 2:
diff --git a/saqc/lib/types.py b/saqc/lib/types.py
index 4c8200b3b..685b06a1e 100644
--- a/saqc/lib/types.py
+++ b/saqc/lib/types.py
@@ -27,10 +27,10 @@ from saqc.core.lib import SaQCFunction, ColumnSelector, APIController
T = TypeVar("T")
ArrayLike = TypeVar("ArrayLike", np.ndarray, pd.Series, pd.DataFrame)
-PandasLike = TypeVar("PandasLike", pd.Series, pd.DataFrame, DictOfSeries)
+PandasLike = Union[pd.Series, pd.DataFrame, DictOfSeries]
DiosLikeT = Union[DictOfSeries, pd.DataFrame]
-FuncReturnT = [DictOfSeries, Flags]
+FuncReturnT = Tuple[DictOfSeries, Flags]
ExternalFlag = Union[str, float, int]
--
GitLab
From 1b13fe3993b2d2ede676c465b24d49f25bc2a888 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 1 Apr 2021 10:52:15 +0200
Subject: [PATCH 110/180] type hints
---
tests/core/test_flags.py | 20 +++++++++++---------
1 file changed, 11 insertions(+), 9 deletions(-)
diff --git a/tests/core/test_flags.py b/tests/core/test_flags.py
index 79445b487..9acdb7f43 100644
--- a/tests/core/test_flags.py
+++ b/tests/core/test_flags.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
+from typing import Dict, Union
import dios
import pytest
import numpy as np
@@ -46,8 +47,9 @@ for d in _data:
@pytest.mark.parametrize('data', data)
-def test_init(data: np.array):
+def test_init(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
flags = Flags(data)
+ import pdb; pdb.set_trace()
assert isinstance(flags, Flags)
assert len(data.keys()) == len(flags)
@@ -59,7 +61,7 @@ def is_equal(f1, f2):
@pytest.mark.parametrize('data', data)
-def test_copy(data: np.array):
+def test_copy(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
flags = Flags(data)
shallow = flags.copy(deep=False)
deep = flags.copy(deep=True)
@@ -83,7 +85,7 @@ def test_copy(data: np.array):
@pytest.mark.parametrize('data', data)
-def test_flags_history(data: np.array):
+def test_flags_history(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
flags = Flags(data)
# get
@@ -103,7 +105,7 @@ def test_flags_history(data: np.array):
@pytest.mark.parametrize('data', data)
-def test_get_flags(data: np.array):
+def test_get_flags(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
flags = Flags(data)
for c in flags.columns:
@@ -122,7 +124,7 @@ def test_get_flags(data: np.array):
@pytest.mark.parametrize('data', data)
-def test_set_flags(data: np.array):
+def test_set_flags(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
flags = Flags(data)
for c in flags.columns:
@@ -150,7 +152,7 @@ def test_set_flags(data: np.array):
assert all(flags.history[c].max() == 8888.)
-@pytest.mark.parametrize('data', data)
+@pytest.mark.parametrize('data', Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]])
def test_set_flags_with_mask(data: np.array):
flags = Flags(data)
@@ -196,7 +198,7 @@ def test_set_flags_with_mask(data: np.array):
@pytest.mark.parametrize('data', data)
-def test_set_flags_with_index(data: np.array):
+def test_set_flags_with_index(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
flags = Flags(data)
for c in flags.columns:
@@ -269,7 +271,7 @@ def _validate_flags_equals_frame(flags, df):
@pytest.mark.parametrize('data', data)
-def test_to_dios(data: np.array):
+def test_to_dios(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
flags = Flags(data)
df = flags.toDios()
@@ -278,7 +280,7 @@ def test_to_dios(data: np.array):
@pytest.mark.parametrize('data', data)
-def test_to_frame(data: np.array):
+def test_to_frame(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
flags = Flags(data)
df = flags.toFrame()
--
GitLab
From 1e96295bb5a79970aab5bd662f235b47ce84c871 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 1 Apr 2021 10:52:43 +0200
Subject: [PATCH 111/180] small fixes to comments, error messages and type
hints
---
saqc/core/flags.py | 16 +++++++++-------
tests/core/test_flags.py | 5 ++---
tests/core/test_history.py | 1 -
3 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index 1cd7d1481..f9ecaa827 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python
from __future__ import annotations
+from dios.dios import DictOfSeries
import pandas as pd
import dios
@@ -108,7 +109,7 @@ class Flags:
As we see above, the column now holds a combination from the values from the
first and the second set. This is, because the special constant ``UNTOUCHED``,
- an alias for ``numpy.nan`` was used. We can inspect all the updates that was
+ an alias for ``numpy.nan`` was used. We can inspect all the updates that was
made by looking in the history.
>>> flags.history['v0']
@@ -320,7 +321,10 @@ class Flags:
# a high potential, that this is not intended by the user.
# if desired use ``flags[:, field] = flag``
if not isinstance(value, pd.Series):
- raise ValueError("must pass value of type pd.Series")
+ raise ValueError(
+ "expected a value of type 'pd.Series', "
+ "if a scalar should be set, please use 'flags[:, field] = flag'"
+ )
# if nothing happens no-one writes the history books
if len(value) == 0:
@@ -482,7 +486,7 @@ def initFlagsLike(
if name is None:
name = reference.name
if name is None:
- raise ValueError("Either the passed series must be named or a name must be passed")
+ raise ValueError("either the passed pd.Series must be named or a name must be passed")
if not isinstance(name, str):
raise TypeError(f"name must be str not '{type(name).__name__}'")
reference = reference.to_frame(name=name)
@@ -490,11 +494,9 @@ def initFlagsLike(
for k, item in reference.items():
if not isinstance(k, str):
- raise TypeError(f"cannot use {k} as key, currently only string keys are allowed")
-
+ raise TypeError(f"cannot use '{k}' as a column name, currently only string keys are allowed")
if k in result:
- raise ValueError('reference must not have duplicate keys')
-
+ raise ValueError('reference must not have duplicate column names')
if not isinstance(item, (pd.Series, History)):
raise TypeError('items in reference must be of type pd.Series')
diff --git a/tests/core/test_flags.py b/tests/core/test_flags.py
index 9acdb7f43..8877cae6b 100644
--- a/tests/core/test_flags.py
+++ b/tests/core/test_flags.py
@@ -49,7 +49,6 @@ for d in _data:
@pytest.mark.parametrize('data', data)
def test_init(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
flags = Flags(data)
- import pdb; pdb.set_trace()
assert isinstance(flags, Flags)
assert len(data.keys()) == len(flags)
@@ -152,8 +151,8 @@ def test_set_flags(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Ser
assert all(flags.history[c].max() == 8888.)
-@pytest.mark.parametrize('data', Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]])
-def test_set_flags_with_mask(data: np.array):
+@pytest.mark.parametrize('data', data)
+def test_set_flags_with_mask(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
flags = Flags(data)
for c in flags.columns:
diff --git a/tests/core/test_history.py b/tests/core/test_history.py
index 1ae95c9db..3b3f87a58 100644
--- a/tests/core/test_history.py
+++ b/tests/core/test_history.py
@@ -103,7 +103,6 @@ def check_invariants(hist):
# or the entire row is True
if not hist.empty:
idxmax = hist.mask.idxmax(axis=1)
- print(f'idxmax: {idxmax}')
for row, col in idxmax.items():
# this is contra intuitive, it gets the positional (for iloc)
row = idxmax.index.get_loc(row)
--
GitLab
From fc60ad9cc6f2dda185e761321e87a206712cbb80 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 1 Apr 2021 12:39:55 +0200
Subject: [PATCH 112/180] fix issues, when pre-exiting flags are passed
---
saqc/core/translator.py | 13 ++++---
tests/core/test_translator.py | 65 +++++++++++++++++++++++++++++++++++
2 files changed, 73 insertions(+), 5 deletions(-)
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
index 327853d76..6f04b4611 100644
--- a/saqc/core/translator.py
+++ b/saqc/core/translator.py
@@ -112,11 +112,8 @@ class DmpTranslator(Translator):
"""
return the names of all functions called on `field`
- NOTE:
- we prepend an empty string to handle variables that
- where never tested
"""
- return [""] + [f.name for l, f in call_stack if l.field == field]
+ return [f.name for l, f in call_stack if l.field == field]
def forward(self, flags: pd.DataFrame) -> Flags:
cols = flags.columns
@@ -132,8 +129,14 @@ class DmpTranslator(Translator):
tflags = super().backward(flags, call_stack)
out = {}
for field in tflags.columns:
- flag_pos = flags.history[field].idxmax()
+ flag_history = flags.history[field]
+ flag_pos = flag_history.idxmax()
flag_funcs = self._getFieldFunctions(field, call_stack)
+ # NOTE:
+ # we prepend empty strings to handle default columns in `Flags`
+ # and potentially given flags not generated during the saqc run,
+ # represented by `call_stack`
+ flag_funcs = ([""] * (len(flag_history.hist.columns) - len(flag_funcs))) + flag_funcs
var_flags = {
self._COL_LABELS["flag"] : tflags[field],
self._COL_LABELS["comment"] : flag_pos.apply(lambda p: json.dumps({"test": flag_funcs[p]})),
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index 949c546cb..1e67a1fe7 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -159,3 +159,68 @@ def test_dmpTranslatorIntegration():
.applymap(lambda v: json.loads(v)["test"] == "")
.all(axis=None))
+
+def _buildupSaQCObjects():
+
+ """
+ return two saqc object, whereas the flags from the previous run
+ are reused
+ """
+ data = initData(3)
+ col = data.columns[0]
+ flags = None
+
+ out = []
+ for _ in range(2):
+ saqc = SaQC(data=data, flags=flags)
+ saqc = (saqc
+ .breaks.flagMissing(col, to_mask=False)
+ .outliers.flagRange(col, min=3, max=10, to_mask=False))
+ saqc = saqc.evaluate()
+ flags = saqc._flags
+ out.append(saqc)
+ return out
+
+
+def test_translationPreservesFlags():
+
+ saqc1, saqc2 = _buildupSaQCObjects()
+ _, flags1 = saqc1.getResult(raw=True)
+ _, flags2 = saqc2.getResult(raw=True)
+
+ for k in flags2.columns:
+ got = flags2.history[k].hist.iloc[:, 1:]
+
+ f1hist = flags1.history[k].hist.iloc[:, 1:]
+ expected = pd.concat([f1hist, f1hist], axis="columns")
+ expected.columns = got.columns
+
+ assert expected.equals(got)
+
+
+def test_positionalTranslationPreservesFlags():
+
+ saqc1, saqc2 = _buildupSaQCObjects()
+ translator = PositionalTranslator()
+ _, flags1 = saqc1.getResult(raw=True)
+ _, flags2 = saqc2.getResult(raw=True)
+ tflags1 = translator.backward(flags1, saqc1._called)
+ tflags2 = translator.backward(flags2, saqc2._called)
+
+ for k in flags2.columns:
+ expected = tflags1[k].str.slice(start=1) * 2
+ got = tflags2[k].str.slice(start=1)
+ assert expected.equals(got)
+
+def test_dmpTranslationPreservesFlags():
+
+ saqc1, saqc2 = _buildupSaQCObjects()
+
+ _, flags1 = saqc1.getResult(raw=True)
+ _, flags2 = saqc2.getResult(raw=True)
+
+ translator = DmpTranslator()
+ tflags1 = translator.backward(flags1, saqc1._called)
+ tflags2 = translator.backward(flags2, saqc2._called)
+
+ assert tflags1.equals(tflags2)
--
GitLab
From 0a8f386e2f9f725b4f7707742a4e9b5e9fb15ca4 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 1 Apr 2021 16:35:19 +0200
Subject: [PATCH 113/180] added docstrings
---
saqc/core/translator.py | 148 ++++++++++++++++++++++++++++++++++++++--
1 file changed, 143 insertions(+), 5 deletions(-)
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
index 6f04b4611..73db0c371 100644
--- a/saqc/core/translator.py
+++ b/saqc/core/translator.py
@@ -47,6 +47,21 @@ class Translator:
TO_MASK = True
def __init__(self, forward: ForwardMap, backward: Optional[BackwardMap]=None):
+ """
+ Parameters
+ ----------
+ forward : dict
+ A mapping defining the forward translation of scalar flag values
+
+ backward : dict, optinal
+ A mapping defining the backward translation of scalar flag values.
+ If not given, `backward` is inferred from `forward`
+
+ Note
+ ----
+ `backward` needs to provide a mappinf for the two special flags
+ `saqc.core.UNFLAGGED`, `saqc.core.BAD`
+ """
# NOTE: we also add the keys to also allow the usage of internal flags
self._forward = forward
if backward is None:
@@ -57,6 +72,19 @@ class Translator:
@staticmethod
def _translate(flags: Union[Flags, pd.DataFrame], trans_map: Union[ForwardMap, BackwardMap]) -> DictOfSeries:
+ """
+ Translate a given flag data structure to another one according to the
+ mapping given in `trans_map`
+
+ Parameters
+ ----------
+ flags : Flags, pd.DataFrame
+ The flags to translate
+
+ Returns
+ -------
+ pd.DataFrame, Flags
+ """
out = DictOfSeries()
expected = pd.Index(trans_map.values())
for field in flags.columns:
@@ -69,6 +97,18 @@ class Translator:
return out
def __call__(self, flag: ExternalFlag) -> float:
+ """
+ Translate a scalar 'external flag' to an 'internal flag'
+
+ Parameters
+ ----------
+ flag : float, int, str
+ The external flag to translate
+
+ Returns
+ -------
+ float
+ """
if flag not in self._forward:
if flag not in self._backward:
raise ValueError(f"invalid flag: {flag}")
@@ -77,15 +117,35 @@ class Translator:
def forward(self, flags: pd.DataFrame) -> Flags:
"""
- translate from 'user flags' to 'internal flags'
+ Translate from 'extrnal flags' to 'internal flags'
+
+ Parameters
+ ----------
+ flags : pd.DataFrame
+ The external flags to translate
+
+ Returns
+ -------
+ Flags object
"""
return Flags(self._translate(flags, self._forward))
def backward(self, flags: Flags, call_stack: CalledStack) -> pd.DataFrame:
"""
- translate from 'internal flags' to 'user flags', the `call_stack`
- ensures access to the history of function called on the associated
- `SaQC`-object.
+ Translate from 'internal flags' to 'external flags'
+
+ Parameters
+ ----------
+ flags : pd.DataFrame
+ The external flags to translate
+ call_stack : List
+ The saqc functions called to generate the given `flags` (i.e. `SaQC._called`)
+ `call_stack` is not evaluated here, it's presence only ensures, that subclasses
+ have access to it.
+
+ Returns
+ -------
+ pd.DataFrame
"""
# NOTE:
return self._translate(flags, self._backward).to_df()
@@ -93,6 +153,11 @@ class Translator:
class FloatTranslator(Translator):
+ """
+ Acts as the default Translator, provides a changeable subset of the
+ internal float flags
+ """
+
_FORWARD: Dict[float, float] = {-np.inf: -np.inf, **{k: k for k in np.arange(0, 256, dtype=float)}}
def __init__(self):
@@ -101,6 +166,11 @@ class FloatTranslator(Translator):
class DmpTranslator(Translator):
+ """
+ Implements the translation from and to the flagging scheme implemented in
+ the UFZ - Datamanagementportal
+ """
+
_FORWARD: Dict[str, float] = {"NIL": UNFLAGGED, "OK": GOOD, "DOUBTFUL": DOUBTFUL, "BAD": BAD}
_COL_LABELS: Dict[str, str] = {"flag": "quality_flag", "comment": "quality_comment", "cause": "quality_cause"}
@@ -110,12 +180,35 @@ class DmpTranslator(Translator):
@staticmethod
def _getFieldFunctions(field: str, call_stack: CalledStack) -> List[str]:
"""
- return the names of all functions called on `field`
+ Return the names of all functions called on `field`
+ Parameters
+ ----------
+ field: str
+ variable/column name
+
+ call_stack : List
+ The saqc functions called to generate the given `flags` (i.e. `SaQC._called`)
+
+ Note
+ ----
+ Could (and maybe should) be implemented as a method of `CalledStack`
"""
return [f.name for l, f in call_stack if l.field == field]
def forward(self, flags: pd.DataFrame) -> Flags:
+ """
+ Translate from 'extrnal flags' to 'internal flags'
+
+ Parameters
+ ----------
+ flags : pd.DataFrame
+ The external flags to translate
+
+ Returns
+ -------
+ Flags object
+ """
cols = flags.columns
if not isinstance(cols, pd.MultiIndex):
raise TypeError("DMP-Flags need mult-index columns")
@@ -126,6 +219,20 @@ class DmpTranslator(Translator):
return super().forward(qflags) # type: ignore
def backward(self, flags: Flags, call_stack: CalledStack) -> pd.DataFrame:
+ """
+ Translate from 'internal flags' to 'external flags'
+
+ Parameters
+ ----------
+ flags : pd.DataFrame
+ The external flags to translate
+ call_stack : List
+ The saqc functions called to generate the given `flags` (i.e. `SaQC._called`)
+
+ Returns
+ -------
+ pd.DataFrame
+ """
tflags = super().backward(flags, call_stack)
out = {}
for field in tflags.columns:
@@ -148,6 +255,10 @@ class DmpTranslator(Translator):
class PositionalTranslator(Translator):
+ """
+ Implements the translation from and to the flagging scheme implemented by CHS
+ """
+
_FORWARD: Dict[int, float] = {0: UNFLAGGED, 1: DOUBTFUL, 2: BAD}
_BACKWARD: Dict[float, int] = {UNTOUCHED: 0, UNFLAGGED: 0, GOOD: 0, DOUBTFUL: 1, BAD: 2}
@@ -155,6 +266,18 @@ class PositionalTranslator(Translator):
super().__init__(forward=self._FORWARD, backward=self._BACKWARD)
def forward(self, flags: pd.DataFrame) -> Flags:
+ """
+ Translate from 'extrnal flags' to 'internal flags'
+
+ Parameters
+ ----------
+ flags : pd.DataFrame
+ The external flags to translate
+
+ Returns
+ -------
+ Flags object
+ """
data = {}
for field in flags.columns:
# drop the first column (i.e. the '9')
@@ -169,6 +292,21 @@ class PositionalTranslator(Translator):
return Flags(data)
def backward(self, flags: Flags, call_stack: CalledStack) -> pd.DataFrame:
+ """
+ Translate from 'internal flags' to 'external flags'
+
+ Parameters
+ ----------
+ flags : pd.DataFrame
+ The external flags to translate
+ call_stack : List
+ The saqc functions called to generate the given `flags` (i.e. `SaQC._called`)
+ `call_stack` is not evaluated here.
+
+ Returns
+ -------
+ pd.DataFrame
+ """
out = {}
for field in flags.columns:
thist = flags.history[field].hist.replace(self._BACKWARD)
--
GitLab
From c30de488a5ad72cdde473a079317a0f97fa13e05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?David=20Sch=C3=A4fer?= <david.schaefer@ufz.de>
Date: Thu, 1 Apr 2021 14:36:58 +0000
Subject: [PATCH 114/180] Apply 1 suggestion(s) to 1 file(s)
---
saqc/core/register.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 06f0df7aa..f0d56b58e 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -169,7 +169,7 @@ def _getMaskingColumns(data: dios.DictOfSeries, field: str, masking: MaskingStrT
def _getMaskingThresh(masking, kwargs, fname):
"""
- Check the correct usage of the `to_mask` keyword, if passed, otherwise return a default.
+ Check the correct usage of the `to_mask` keyword, iff passed, otherwise return a default.
Parameters
----------
--
GitLab
From d0cd8d6cf2eafcc6a5dc89df78aa9ae3b4a3aa57 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Tue, 13 Apr 2021 12:24:39 +0200
Subject: [PATCH 115/180] documentation and refactoring
---
saqc/core/core.py | 26 ++++++++++++++++++++------
1 file changed, 20 insertions(+), 6 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 70ac3e2e3..8a9209e0b 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -117,7 +117,8 @@ class SaQC(FuncModules):
@staticmethod
def _initFlags(data, flags: Optional[Flags]):
- """ Init the internal Flags-object.
+ """
+ Init the internal Flags-object.
Ensures that all data columns are present and user passed
flags from a frame or an already initialised Flags-object
@@ -132,7 +133,20 @@ class SaQC(FuncModules):
return flags
- def _constructSimple(self, **injectables) -> SaQC:
+ def _construct(self, **injectables) -> SaQC:
+ """
+ Construct a new `SaQC`-Object from `self` and optionally inject
+ attributes with any chechking and overhead.
+
+ Parameters
+ ----------
+ **injectables: any of the `SaQC` data attributes with name and value
+
+ Note
+ ----
+ For internal usage only! Setting values through `injectables` has
+ the potential to mess up certain invariants of the constructed object.
+ """
out = SaQC(
data=DictOfSeries(),
flags=Flags(),
@@ -142,7 +156,7 @@ class SaQC(FuncModules):
)
for k, v in injectables.items():
if not hasattr(out, k):
- raise ValueError(f"failed to set unknown attribute: {k}")
+ raise AttributeError(f"failed to set unknown attribute: {k}")
setattr(out, k, v)
return out
@@ -166,8 +180,8 @@ class SaQC(FuncModules):
"""
Realize all the registered calculations and return a updated SaQC Object
- Paramters
- ---------
+ Parameters
+ ----------
Returns
-------
@@ -208,7 +222,7 @@ class SaQC(FuncModules):
# This is way faster for big datasets, than to throw everything in the constructor.
# Simply because of _initFlags -> merge() -> mergeDios() over all columns.
- return self._constructSimple(
+ return self._construct(
_flags=flags, _data=data, _called=self._called + called
)
--
GitLab
From 0cca182ba3d632fb55c322188e2680f870a6e932 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Tue, 13 Apr 2021 14:16:49 +0200
Subject: [PATCH 116/180] commenting and renaming the function call lists
---
saqc/core/core.py | 27 ++++++++++++++++-----------
saqc/core/translator.py | 18 +++++++++---------
saqc/lib/types.py | 6 +++---
tests/core/test_reader.py | 6 +++---
tests/core/test_translator.py | 12 ++++++------
5 files changed, 37 insertions(+), 32 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 8a9209e0b..63db1a4ec 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -24,7 +24,7 @@ from saqc.core.modules import FuncModules
from saqc.funcs.tools import copy
from saqc.lib.plotting import plotHook, plotAllHook
from saqc.core.translator import FloatTranslator, Translator
-from saqc.lib.types import ExternalFlag, CallStack, CalledStack, FuncReturnT, PandasLike
+from saqc.lib.types import ExternalFlag, CallGraph, MaterializedGraph, PandasLike
from saqc.constants import BAD
logger = logging.getLogger("SaQC")
@@ -112,8 +112,13 @@ class SaQC(FuncModules):
self._flags = self._initFlags(data, flags)
self._error_policy = error_policy
self._translator = translator or FloatTranslator()
- self._to_call: CallStack = [] # will be filled by calls to `_wrap`
- self._called: CalledStack = [] # will be filled in `evaluate`
+ # NOTE:
+ # We need two lists to represent the future and the past computations
+ # on a `SaQC`-Object. Due to the dynamic nature of field expansion
+ # with regular expressions, we can't just reuse the original execution
+ # plan to infer all translation related information.
+ self._planned: CallGraph = [] # will be filled by calls to `_wrap`
+ self._computed: MaterializedGraph = [] # will be filled in `evaluate`
@staticmethod
def _initFlags(data, flags: Optional[Flags]):
@@ -163,7 +168,7 @@ class SaQC(FuncModules):
def readConfig(self, fname):
from saqc.core.reader import readConfig
out = stdcopy.deepcopy(self)
- out._to_call.extend(readConfig(fname, self._flags, self._nodata))
+ out._planned.extend(readConfig(fname, self._flags, self._nodata))
return out
@staticmethod
@@ -191,14 +196,14 @@ class SaQC(FuncModules):
# NOTE: It would be nicer to separate the plotting into an own
# method instead of intermingling it with the computation
data, flags = self._data, self._flags
- called: CalledStack = []
- for selector, control, function in self._to_call:
+ computed: MaterializedGraph = []
+ for selector, control, function in self._planned:
for sel, func in self._expandFields(selector, function, data.columns.union(flags.columns)):
logger.debug(f"processing: {sel.field}, {func.name}, {func.keywords}")
try:
data_result, flags_result = _saqcCallFunc(sel, control, func, data, flags)
- called.append((sel, func))
+ computed.append((sel, func))
except Exception as e:
_handleErrors(e, sel.field, control, func, self._error_policy)
continue
@@ -217,13 +222,13 @@ class SaQC(FuncModules):
data = data_result
flags = flags_result
- if any([control.plot for _, control, _ in self._to_call]):
+ if any([control.plot for _, control, _ in self._planned]):
plotAllHook(data, flags)
# This is way faster for big datasets, than to throw everything in the constructor.
# Simply because of _initFlags -> merge() -> mergeDios() over all columns.
return self._construct(
- _flags=flags, _data=data, _called=self._called + called
+ _flags=flags, _data=data, _computed=self._computed + computed
)
def getResult(self, raw=False) -> Union[Tuple[DictOfSeries, Flags], Tuple[pd.DataFrame, pd.DataFrame]]:
@@ -241,7 +246,7 @@ class SaQC(FuncModules):
if raw:
return data, flags
- return data.to_df(), self._translator.backward(flags, realization._called)
+ return data.to_df(), self._translator.backward(flags, realization._computed)
def _wrap(self, func: SaQCFunction):
@@ -265,7 +270,7 @@ class SaQC(FuncModules):
)
out = self if inplace else self.copy(deep=True)
- out._to_call.append((locator, control, partial))
+ out._planned.append((locator, control, partial))
return out
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
index 73db0c371..b8418aafc 100644
--- a/saqc/core/translator.py
+++ b/saqc/core/translator.py
@@ -13,7 +13,7 @@ from dios import DictOfSeries
from saqc.core.flags import Flags, UNTOUCHED, UNFLAGGED, GOOD, DOUBTFUL, BAD
from saqc.core.history import History
-from saqc.lib.types import ExternalFlag, CalledStack
+from saqc.lib.types import ExternalFlag, MaterializedGraph
# to_mask as part of th translator
@@ -130,7 +130,7 @@ class Translator:
"""
return Flags(self._translate(flags, self._forward))
- def backward(self, flags: Flags, call_stack: CalledStack) -> pd.DataFrame:
+ def backward(self, flags: Flags, call_stack: MaterializedGraph) -> pd.DataFrame:
"""
Translate from 'internal flags' to 'external flags'
@@ -139,7 +139,7 @@ class Translator:
flags : pd.DataFrame
The external flags to translate
call_stack : List
- The saqc functions called to generate the given `flags` (i.e. `SaQC._called`)
+ The saqc functions called to generate the given `flags` (i.e. `SaQC._computed`)
`call_stack` is not evaluated here, it's presence only ensures, that subclasses
have access to it.
@@ -178,7 +178,7 @@ class DmpTranslator(Translator):
super().__init__(forward=self._FORWARD)
@staticmethod
- def _getFieldFunctions(field: str, call_stack: CalledStack) -> List[str]:
+ def _getFieldFunctions(field: str, call_stack: MaterializedGraph) -> List[str]:
"""
Return the names of all functions called on `field`
@@ -188,7 +188,7 @@ class DmpTranslator(Translator):
variable/column name
call_stack : List
- The saqc functions called to generate the given `flags` (i.e. `SaQC._called`)
+ The saqc functions called to generate the given `flags` (i.e. `SaQC._computed`)
Note
----
@@ -218,7 +218,7 @@ class DmpTranslator(Translator):
qflags = flags.xs(key=self._COL_LABELS["flag"], axis="columns", level=1)
return super().forward(qflags) # type: ignore
- def backward(self, flags: Flags, call_stack: CalledStack) -> pd.DataFrame:
+ def backward(self, flags: Flags, call_stack: MaterializedGraph) -> pd.DataFrame:
"""
Translate from 'internal flags' to 'external flags'
@@ -227,7 +227,7 @@ class DmpTranslator(Translator):
flags : pd.DataFrame
The external flags to translate
call_stack : List
- The saqc functions called to generate the given `flags` (i.e. `SaQC._called`)
+ The saqc functions called to generate the given `flags` (i.e. `SaQC._computed`)
Returns
-------
@@ -291,7 +291,7 @@ class PositionalTranslator(Translator):
data[field] = tflags
return Flags(data)
- def backward(self, flags: Flags, call_stack: CalledStack) -> pd.DataFrame:
+ def backward(self, flags: Flags, call_stack: MaterializedGraph) -> pd.DataFrame:
"""
Translate from 'internal flags' to 'external flags'
@@ -300,7 +300,7 @@ class PositionalTranslator(Translator):
flags : pd.DataFrame
The external flags to translate
call_stack : List
- The saqc functions called to generate the given `flags` (i.e. `SaQC._called`)
+ The saqc functions called to generate the given `flags` (i.e. `SaQC._computed`)
`call_stack` is not evaluated here.
Returns
diff --git a/saqc/lib/types.py b/saqc/lib/types.py
index 685b06a1e..fd8352fc1 100644
--- a/saqc/lib/types.py
+++ b/saqc/lib/types.py
@@ -37,15 +37,15 @@ ExternalFlag = Union[str, float, int]
# we only support fixed length offsets
FreqString = NewType("FreqString", Literal["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"])
-CallStack = List[Tuple[ColumnSelector, APIController, SaQCFunction]]
-CalledStack = List[Tuple[ColumnSelector, Optional[SaQCFunction]]]
+CallGraph = List[Tuple[ColumnSelector, APIController, SaQCFunction]]
+MaterializedGraph = List[Tuple[ColumnSelector, Optional[SaQCFunction]]]
# we define a bunch of type aliases, mostly needed to generate appropiate fuzzy data through hypothesis
ColumnName = NewType("ColumnName", str)
IntegerWindow = NewType("IntegerWindow", int)
TimestampColumnName = TypeVar("TimestampColumnName", bound=str)
-# needed for deeper typy hinting magic
+# needed for deeper type hinting magic
class CurveFitter(Protocol):
def __call__(self, data: np.ndarray, *params: float) -> np.ndarray:
...
diff --git a/tests/core/test_reader.py b/tests/core/test_reader.py
index ded1bdf82..dd1941cf8 100644
--- a/tests/core/test_reader.py
+++ b/tests/core/test_reader.py
@@ -45,7 +45,7 @@ def test_variableRegex(data):
for regex, expected in tests:
fobj = writeIO(header + "\n" + f"{regex} ; flagtools.flagDummy()")
saqc = SaQC(data).readConfig(fobj)
- expansion = saqc._expandFields(saqc._to_call[0][0], saqc._to_call[0][2], data.columns)
+ expansion = saqc._expandFields(saqc._planned[0][0], saqc._planned[0][2], data.columns)
result = [s.field for s, _ in expansion]
assert np.all(result == expected)
@@ -59,7 +59,7 @@ def test_inlineComments(data):
pre2 ; flagtools.flagDummy() # test ; False # test
"""
saqc = SaQC(data).readConfig(writeIO(config))
- _, control, func = saqc._to_call[0]
+ _, control, func = saqc._planned[0]
assert control.plot is False
assert func.func == FUNC_MAP["flagtools.flagDummy"].func
@@ -77,7 +77,7 @@ def test_configReaderLineNumbers(data):
SM1 ; flagtools.flagDummy()
"""
saqc = SaQC(data).readConfig(writeIO(config))
- result = [c.lineno for _, c, _ in saqc._to_call]
+ result = [c.lineno for _, c, _ in saqc._planned]
expected = [3, 4, 5, 9]
assert result == expected
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index 1e67a1fe7..6fddf9a6d 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -125,7 +125,7 @@ def test_positionalTranslatorIntegration():
for field in flags.columns:
assert flags[field].str.match("^9[012]*$").all()
- round_trip = translator.backward(translator.forward(flags), saqc._called)
+ round_trip = translator.backward(translator.forward(flags), saqc._computed)
assert (flags.values == round_trip.values).all()
assert (flags.index == round_trip.index).all()
@@ -152,7 +152,7 @@ def test_dmpTranslatorIntegration():
assert qfunc.isin({"", "breaks.flagMissing", "outliers.flagRange"}).all(axis=None)
assert (qcause == "").all(axis=None)
- round_trip = translator.backward(translator.forward(flags), saqc._called)
+ round_trip = translator.backward(translator.forward(flags), saqc._computed)
assert round_trip.xs("quality_flag", axis="columns", level=1).equals(qflags)
assert (round_trip
.xs("quality_comment", axis="columns", level=1)
@@ -204,8 +204,8 @@ def test_positionalTranslationPreservesFlags():
translator = PositionalTranslator()
_, flags1 = saqc1.getResult(raw=True)
_, flags2 = saqc2.getResult(raw=True)
- tflags1 = translator.backward(flags1, saqc1._called)
- tflags2 = translator.backward(flags2, saqc2._called)
+ tflags1 = translator.backward(flags1, saqc1._computed)
+ tflags2 = translator.backward(flags2, saqc2._computed)
for k in flags2.columns:
expected = tflags1[k].str.slice(start=1) * 2
@@ -220,7 +220,7 @@ def test_dmpTranslationPreservesFlags():
_, flags2 = saqc2.getResult(raw=True)
translator = DmpTranslator()
- tflags1 = translator.backward(flags1, saqc1._called)
- tflags2 = translator.backward(flags2, saqc2._called)
+ tflags1 = translator.backward(flags1, saqc1._computed)
+ tflags2 = translator.backward(flags2, saqc2._computed)
assert tflags1.equals(tflags2)
--
GitLab
From 71355928aa60278536d6dd4aa934839a49d1a8eb Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Tue, 13 Apr 2021 21:55:44 +0200
Subject: [PATCH 117/180] black formatting
---
saqc/__main__.py | 54 ++-
saqc/constants.py | 6 +-
saqc/core/core.py | 92 +++--
saqc/core/flags.py | 46 ++-
saqc/core/history.py | 60 +--
saqc/core/lib.py | 14 +-
saqc/core/modules/base.py | 1 -
saqc/core/modules/breaks.py | 33 +-
saqc/core/modules/changepoints.py | 63 +--
saqc/core/modules/constants.py | 29 +-
saqc/core/modules/curvefit.py | 20 +-
saqc/core/modules/drift.py | 111 +++---
saqc/core/modules/flagtools.py | 24 +-
saqc/core/modules/generic.py | 23 +-
saqc/core/modules/interpolation.py | 48 +--
saqc/core/modules/outliers.py | 147 ++++---
saqc/core/modules/pattern.py | 29 +-
saqc/core/modules/resampling.py | 116 +++---
saqc/core/modules/residues.py | 37 +-
saqc/core/modules/rolling.py | 20 +-
saqc/core/modules/scores.py | 29 +-
saqc/core/modules/tools.py | 16 +-
saqc/core/modules/transformation.py | 11 +-
saqc/core/reader.py | 8 +-
saqc/core/register.py | 57 +--
saqc/core/translator.py | 66 +++-
saqc/core/visitor.py | 4 +-
saqc/funcs/breaks.py | 56 +--
saqc/funcs/changepoints.py | 115 +++---
saqc/funcs/constants.py | 40 +-
saqc/funcs/curvefit.py | 67 ++--
saqc/funcs/drift.py | 272 +++++++------
saqc/funcs/flagtools.py | 42 +-
saqc/funcs/generic.py | 49 +--
saqc/funcs/interpolation.py | 87 +++--
saqc/funcs/outliers.py | 390 +++++++++++--------
saqc/funcs/pattern.py | 42 +-
saqc/funcs/resampling.py | 248 ++++++------
saqc/funcs/residues.py | 52 +--
saqc/funcs/rolling.py | 52 ++-
saqc/funcs/scores.py | 40 +-
saqc/funcs/tools.py | 42 +-
saqc/funcs/transformation.py | 14 +-
saqc/lib/plotting.py | 41 +-
saqc/lib/rolling.py | 70 +++-
saqc/lib/tools.py | 108 +++--
saqc/lib/ts_operators.py | 78 ++--
saqc/lib/types.py | 26 +-
tests/common.py | 6 +-
tests/core/test_core.py | 23 +-
tests/core/test_creation.py | 12 +-
tests/core/test_flags.py | 139 +++----
tests/core/test_history.py | 139 +++----
tests/core/test_reader.py | 12 +-
tests/core/test_translator.py | 53 ++-
tests/fixtures.py | 108 ++---
tests/funcs/test_constants_detection.py | 8 +-
tests/funcs/test_functions.py | 103 +++--
tests/funcs/test_generic_api_functions.py | 24 +-
tests/funcs/test_generic_config_functions.py | 23 +-
tests/funcs/test_harm_funcs.py | 213 +++++++---
tests/funcs/test_modelling.py | 60 ++-
tests/funcs/test_pattern_rec.py | 20 +-
tests/funcs/test_proc_functions.py | 70 +++-
tests/funcs/test_spikes_detection.py | 45 ++-
tests/fuzzy/lib.py | 27 +-
tests/fuzzy/test_functions.py | 2 +
tests/fuzzy/test_masking.py | 60 ++-
tests/integration/test_integration.py | 17 +-
tests/lib/test_rolling.py | 66 ++--
70 files changed, 2615 insertions(+), 1710 deletions(-)
diff --git a/saqc/__main__.py b/saqc/__main__.py
index 06a30ff83..1cafc99ed 100644
--- a/saqc/__main__.py
+++ b/saqc/__main__.py
@@ -37,13 +37,15 @@ def _setupLogging(loglvl):
def setupIO(nodata):
reader = {
- ".csv" : partial(pd.read_csv, index_col=0, parse_dates=True),
- ".parquet" : pd.read_parquet
+ ".csv": partial(pd.read_csv, index_col=0, parse_dates=True),
+ ".parquet": pd.read_parquet,
}
writer = {
- ".csv" : partial(pd.DataFrame.to_csv, header=True, index=True, na_rep=nodata),
- ".parquet" : lambda df, outfile: pa.parquet.write_table(pa.Table.from_pandas(df), outfile)
+ ".csv": partial(pd.DataFrame.to_csv, header=True, index=True, na_rep=nodata),
+ ".parquet": lambda df, outfile: pa.parquet.write_table(
+ pa.Table.from_pandas(df), outfile
+ ),
}
return reader, writer
@@ -52,7 +54,9 @@ def readData(reader_dict, fname):
extension = Path(fname).suffix
reader = reader_dict.get(extension)
if not reader:
- raise ValueError(f"Unsupported file format '{extension}', use one of {tuple(reader.keys())}")
+ raise ValueError(
+ f"Unsupported file format '{extension}', use one of {tuple(reader.keys())}"
+ )
return reader(fname)
@@ -60,26 +64,46 @@ def writeData(writer_dict, df, fname):
extension = Path(fname).suffix
writer = writer_dict.get(extension)
if not writer:
- raise ValueError(f"Unsupported file format '{extension}', use one of {tuple(writer.keys())}")
+ raise ValueError(
+ f"Unsupported file format '{extension}', use one of {tuple(writer.keys())}"
+ )
writer(df, fname)
@click.command()
@click.option(
- "-c", "--config", type=click.Path(exists=True), required=True, help="path to the configuration file",
+ "-c",
+ "--config",
+ type=click.Path(exists=True),
+ required=True,
+ help="path to the configuration file",
+)
+@click.option(
+ "-d",
+ "--data",
+ type=click.Path(exists=True),
+ required=True,
+ help="path to the data file",
)
@click.option(
- "-d", "--data", type=click.Path(exists=True), required=True, help="path to the data file",
+ "-o", "--outfile", type=click.Path(exists=False), help="path to the output file"
)
-@click.option("-o", "--outfile", type=click.Path(exists=False), help="path to the output file")
@click.option(
- "--flagger", default=None, type=click.Choice(SCHEMES.keys()), help="the flagging scheme to use",
+ "--flagger",
+ default=None,
+ type=click.Choice(SCHEMES.keys()),
+ help="the flagging scheme to use",
)
@click.option("--nodata", default=np.nan, help="nodata value")
@click.option(
- "--log-level", default="INFO", type=click.Choice(["DEBUG", "INFO", "WARNING"]), help="set output verbosity"
+ "--log-level",
+ default="INFO",
+ type=click.Choice(["DEBUG", "INFO", "WARNING"]),
+ help="set output verbosity",
+)
+@click.option(
+ "--fail/--no-fail", default=True, help="whether to stop the program run on errors"
)
-@click.option("--fail/--no-fail", default=True, help="whether to stop the program run on errors")
def main(config, data, flagger, outfile, nodata, log_level, fail):
if SCHEMES[flagger] is NotImplemented:
@@ -90,7 +114,11 @@ def main(config, data, flagger, outfile, nodata, log_level, fail):
data = readData(reader, data)
- saqc = SaQC(data=data, nodata=nodata, error_policy="raise" if fail else "warn",)
+ saqc = SaQC(
+ data=data,
+ nodata=nodata,
+ error_policy="raise" if fail else "warn",
+ )
data_result, flags_result = saqc.readConfig(config).getResult(raw=True)
diff --git a/saqc/constants.py b/saqc/constants.py
index 89db985ec..7d4cb5f62 100644
--- a/saqc/constants.py
+++ b/saqc/constants.py
@@ -14,10 +14,8 @@ import numpy as np
UNTOUCHED = np.nan
UNFLAGGED = -np.inf
GOOD = 0
-DOUBTFUL = 25.
-BAD = 255.
+DOUBTFUL = 25.0
+BAD = 255.0
# aliases
DOUBT = DOUBTFUL
-
-
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 63db1a4ec..633ff9b2e 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -30,13 +30,19 @@ from saqc.constants import BAD
logger = logging.getLogger("SaQC")
-def _handleErrors(exc: Exception, field: str, control: APIController, func: SaQCFunction, policy: Literal["ignore", "warn", "raise"]):
+def _handleErrors(
+ exc: Exception,
+ field: str,
+ control: APIController,
+ func: SaQCFunction,
+ policy: Literal["ignore", "warn", "raise"],
+):
message = "\n".join(
[
f"Exception:\n{type(exc).__name__}: {exc}",
f"field: {field}",
f"{func.errorMessage()}",
- f"{control.errorMessage()}"
+ f"{control.errorMessage()}",
]
)
@@ -50,17 +56,23 @@ def _handleErrors(exc: Exception, field: str, control: APIController, func: SaQC
# TODO: shouldt the code/function go to Saqc.__init__ ?
-def _prepInput(data: PandasLike, flags: Optional[Union[DictOfSeries, pd.DataFrame, Flags]]) -> Tuple[DictOfSeries, Optional[Flags]]:
+def _prepInput(
+ data: PandasLike, flags: Optional[Union[DictOfSeries, pd.DataFrame, Flags]]
+) -> Tuple[DictOfSeries, Optional[Flags]]:
dios_like = (DictOfSeries, pd.DataFrame)
if isinstance(data, pd.Series):
data = data.to_frame()
if not isinstance(data, dios_like):
- raise TypeError("'data' must be of type pd.Series, pd.DataFrame or DictOfSeries")
+ raise TypeError(
+ "'data' must be of type pd.Series, pd.DataFrame or DictOfSeries"
+ )
if isinstance(data, pd.DataFrame):
- if isinstance(data.index, pd.MultiIndex) or isinstance(data.columns, pd.MultiIndex):
+ if isinstance(data.index, pd.MultiIndex) or isinstance(
+ data.columns, pd.MultiIndex
+ ):
raise TypeError("'data' should not use MultiIndex")
data = to_dios(data)
@@ -70,7 +82,9 @@ def _prepInput(data: PandasLike, flags: Optional[Union[DictOfSeries, pd.DataFram
if flags is not None:
if isinstance(flags, pd.DataFrame):
- if isinstance(flags.index, pd.MultiIndex) or isinstance(flags.columns, pd.MultiIndex):
+ if isinstance(flags.index, pd.MultiIndex) or isinstance(
+ flags.columns, pd.MultiIndex
+ ):
raise TypeError("'flags' should not use MultiIndex")
if isinstance(flags, (DictOfSeries, pd.DataFrame, Flags)):
@@ -79,7 +93,9 @@ def _prepInput(data: PandasLike, flags: Optional[Union[DictOfSeries, pd.DataFram
cols = flags.columns.intersection(data.columns)
for c in cols:
if not flags[c].index.equals(data[c].index):
- raise ValueError(f"the index of 'flags' and 'data' missmatch in column {c}")
+ raise ValueError(
+ f"the index of 'flags' and 'data' missmatch in column {c}"
+ )
# this also ensures float dtype
if not isinstance(flags, Flags):
@@ -103,8 +119,14 @@ _setup()
class SaQC(FuncModules):
-
- def __init__(self, data, flags=None, translator: Translator=None, nodata=np.nan, error_policy="raise"):
+ def __init__(
+ self,
+ data,
+ flags=None,
+ translator: Translator = None,
+ nodata=np.nan,
+ error_policy="raise",
+ ):
super().__init__(self)
data, flags = _prepInput(data, flags)
self._data = data
@@ -117,7 +139,7 @@ class SaQC(FuncModules):
# on a `SaQC`-Object. Due to the dynamic nature of field expansion
# with regular expressions, we can't just reuse the original execution
# plan to infer all translation related information.
- self._planned: CallGraph = [] # will be filled by calls to `_wrap`
+ self._planned: CallGraph = [] # will be filled by calls to `_wrap`
self._computed: MaterializedGraph = [] # will be filled in `evaluate`
@staticmethod
@@ -167,18 +189,28 @@ class SaQC(FuncModules):
def readConfig(self, fname):
from saqc.core.reader import readConfig
+
out = stdcopy.deepcopy(self)
out._planned.extend(readConfig(fname, self._flags, self._nodata))
return out
@staticmethod
- def _expandFields(selector: ColumnSelector, func: SaQCFunction, variables: pd.Index) -> Sequence[Tuple[ColumnSelector, SaQCFunction]]:
+ def _expandFields(
+ selector: ColumnSelector, func: SaQCFunction, variables: pd.Index
+ ) -> Sequence[Tuple[ColumnSelector, SaQCFunction]]:
if not selector.regex:
return [(selector, func)]
out = []
for field in variables[variables.str.match(selector.field)]:
- out.append((ColumnSelector(field=field, target=selector.target, regex=selector.regex), func))
+ out.append(
+ (
+ ColumnSelector(
+ field=field, target=selector.target, regex=selector.regex
+ ),
+ func,
+ )
+ )
return out
def evaluate(self):
@@ -198,11 +230,15 @@ class SaQC(FuncModules):
data, flags = self._data, self._flags
computed: MaterializedGraph = []
for selector, control, function in self._planned:
- for sel, func in self._expandFields(selector, function, data.columns.union(flags.columns)):
+ for sel, func in self._expandFields(
+ selector, function, data.columns.union(flags.columns)
+ ):
logger.debug(f"processing: {sel.field}, {func.name}, {func.keywords}")
try:
- data_result, flags_result = _saqcCallFunc(sel, control, func, data, flags)
+ data_result, flags_result = _saqcCallFunc(
+ sel, control, func, data, flags
+ )
computed.append((sel, func))
except Exception as e:
_handleErrors(e, sel.field, control, func, self._error_policy)
@@ -231,7 +267,9 @@ class SaQC(FuncModules):
_flags=flags, _data=data, _computed=self._computed + computed
)
- def getResult(self, raw=False) -> Union[Tuple[DictOfSeries, Flags], Tuple[pd.DataFrame, pd.DataFrame]]:
+ def getResult(
+ self, raw=False
+ ) -> Union[Tuple[DictOfSeries, Flags], Tuple[pd.DataFrame, pd.DataFrame]]:
"""
Realize the registered calculations and return the results
@@ -249,14 +287,20 @@ class SaQC(FuncModules):
return data.to_df(), self._translator.backward(flags, realization._computed)
def _wrap(self, func: SaQCFunction):
+ def inner(
+ field: str,
+ *fargs,
+ target: str = None,
+ regex: bool = False,
+ flag: ExternalFlag = BAD,
+ plot: bool = False,
+ inplace: bool = False,
+ **fkwargs,
+ ) -> SaQC:
- def inner(field: str, *fargs, target: str=None, regex: bool=False, flag: ExternalFlag=BAD, plot: bool=False, inplace: bool=False, **fkwargs) -> SaQC:
+ fkwargs.setdefault("to_mask", self._translator.TO_MASK)
- fkwargs.setdefault('to_mask', self._translator.TO_MASK)
-
- control = APIController(
- plot=plot
- )
+ control = APIController(plot=plot)
locator = ColumnSelector(
field=field,
@@ -266,7 +310,7 @@ class SaQC(FuncModules):
partial = func.bind(
*fargs,
- **{"nodata": self._nodata, "flag": self._translator(flag), **fkwargs}
+ **{"nodata": self._nodata, "flag": self._translator(flag), **fkwargs},
)
out = self if inplace else self.copy(deep=True)
@@ -316,7 +360,7 @@ def _saqcCallFunc(locator, controller, function, data, flags):
def _warnForUnusedKwargs(func):
- """ Warn for unused kwargs, passed to a SaQC.function.
+ """Warn for unused kwargs, passed to a SaQC.function.
Parameters
----------
@@ -346,5 +390,5 @@ def _warnForUnusedKwargs(func):
missing.append(kw)
if missing:
- missing = ', '.join(missing)
+ missing = ", ".join(missing)
logging.warning(f"Unused argument(s): {missing}")
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index f9ecaa827..ead06fffb 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -30,7 +30,6 @@ ValueT = Union[pd.Series, Iterable, float]
class _HistAccess:
-
def __init__(self, obj: Flags):
self.obj = obj
@@ -164,7 +163,9 @@ class Flags:
2 (-inf) (25.0) (0.0) 99.0
"""
- def __init__(self, raw_data: Optional[Union[DictLike, Flags]] = None, copy: bool = False):
+ def __init__(
+ self, raw_data: Optional[Union[DictLike, Flags]] = None, copy: bool = False
+ ):
if raw_data is None:
raw_data = {}
@@ -192,7 +193,7 @@ class Flags:
for k, item in data.items():
if k in result:
- raise ValueError('raw_data must not have duplicate keys')
+ raise ValueError("raw_data must not have duplicate keys")
# No, means no ! (copy)
if isinstance(item, History) and not copy:
@@ -206,14 +207,16 @@ class Flags:
elif isinstance(item, (History, pd.DataFrame)):
pass
else:
- raise TypeError(f"cannot init from '{type(data).__name__}' of '{type(item).__name__}'")
+ raise TypeError(
+ f"cannot init from '{type(data).__name__}' of '{type(item).__name__}'"
+ )
result[k] = History(item, copy=copy)
return result
@property
- def _constructor(self) -> Type['Flags']:
+ def _constructor(self) -> Type["Flags"]:
return type(self)
# ----------------------------------------------------------------------
@@ -244,11 +247,8 @@ class Flags:
if not isinstance(value, pd.Index):
value = pd.Index(value)
- if (
- not value.is_unique
- or not pd.api.types.is_string_dtype(value)
- ):
- raise TypeError('value must be pd.Index, with unique indices of type str')
+ if not value.is_unique or not pd.api.types.is_string_dtype(value):
+ raise TypeError("value must be pd.Index, with unique indices of type str")
if not len(value) == len(self):
raise ValueError("index must match current index in length")
@@ -297,7 +297,9 @@ class Flags:
if isinstance(key, tuple):
if len(key) != 2:
- raise KeyError("a single 'column' or a tuple of 'mask, column' must be passt")
+ raise KeyError(
+ "a single 'column' or a tuple of 'mask, column' must be passt"
+ )
mask, key = key
tmp = pd.Series(UNTOUCHED, index=self._data[key].index, dtype=float)
@@ -312,7 +314,7 @@ class Flags:
try:
tmp[mask] = value
except Exception:
- raise ValueError('bad mask')
+ raise ValueError("bad mask")
else:
value = tmp
@@ -438,13 +440,13 @@ class Flags:
return self.toDios().to_df()
def __repr__(self) -> str:
- return str(self.toDios()).replace('DictOfSeries', type(self).__name__)
+ return str(self.toDios()).replace("DictOfSeries", type(self).__name__)
def initFlagsLike(
- reference: Union[pd.Series, DictLike, Flags],
- initial_value: float = UNFLAGGED,
- name: str = None,
+ reference: Union[pd.Series, DictLike, Flags],
+ initial_value: float = UNFLAGGED,
+ name: str = None,
) -> Flags:
"""
Create empty Flags, from an reference data structure.
@@ -486,7 +488,9 @@ def initFlagsLike(
if name is None:
name = reference.name
if name is None:
- raise ValueError("either the passed pd.Series must be named or a name must be passed")
+ raise ValueError(
+ "either the passed pd.Series must be named or a name must be passed"
+ )
if not isinstance(name, str):
raise TypeError(f"name must be str not '{type(name).__name__}'")
reference = reference.to_frame(name=name)
@@ -494,11 +498,13 @@ def initFlagsLike(
for k, item in reference.items():
if not isinstance(k, str):
- raise TypeError(f"cannot use '{k}' as a column name, currently only string keys are allowed")
+ raise TypeError(
+ f"cannot use '{k}' as a column name, currently only string keys are allowed"
+ )
if k in result:
- raise ValueError('reference must not have duplicate column names')
+ raise ValueError("reference must not have duplicate column names")
if not isinstance(item, (pd.Series, History)):
- raise TypeError('items in reference must be of type pd.Series')
+ raise TypeError("items in reference must be of type pd.Series")
item = pd.DataFrame(initial_value, index=item.index, columns=[0], dtype=float)
diff --git a/saqc/core/history.py b/saqc/core/history.py
index 53688d72f..02cbc0c53 100644
--- a/saqc/core/history.py
+++ b/saqc/core/history.py
@@ -47,7 +47,9 @@ class History:
If True, the input data is copied, otherwise not.
"""
- def __init__(self, hist: pd.DataFrame = None, mask: pd.DataFrame = None, copy: bool = False):
+ def __init__(
+ self, hist: pd.DataFrame = None, mask: pd.DataFrame = None, copy: bool = False
+ ):
# this is a hidden _feature_ and not exposed by the type
# of the hist parameter and serve as a fastpath for internal
@@ -330,7 +332,7 @@ class History:
return self.hist[self.mask].max(axis=1)
@property
- def _constructor(self) -> Type['History']:
+ def _constructor(self) -> Type["History"]:
return History
def copy(self, deep=True) -> History:
@@ -390,13 +392,13 @@ class History:
def __repr__(self):
if self.empty:
- return str(self.hist).replace('DataFrame', 'History')
+ return str(self.hist).replace("DataFrame", "History")
repr = self.hist.astype(str)
m = self.mask
- repr[m] = ' ' + repr[m] + ' '
- repr[~m] = '(' + repr[~m] + ')'
+ repr[m] = " " + repr[m] + " "
+ repr[~m] = "(" + repr[~m] + ")"
return str(repr)[1:]
@@ -405,7 +407,9 @@ class History:
#
@staticmethod
- def _validateHistWithMask(obj: pd.DataFrame, mask: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]:
+ def _validateHistWithMask(
+ obj: pd.DataFrame, mask: pd.DataFrame
+ ) -> Tuple[pd.DataFrame, pd.DataFrame]:
"""
check type, columns, index, dtype and if the mask fits the obj.
"""
@@ -415,13 +419,17 @@ class History:
# check mask
if not isinstance(mask, pd.DataFrame):
- raise TypeError(f"'mask' must be of type pd.DataFrame, but {type(mask).__name__} was given")
+ raise TypeError(
+ f"'mask' must be of type pd.DataFrame, but {type(mask).__name__} was given"
+ )
if any(mask.dtypes != bool):
raise ValueError("dtype of all columns in 'mask' must be bool")
if not mask.empty and not mask.iloc[:, -1].all():
- raise ValueError("the values in the last column in mask must be 'True' everywhere.")
+ raise ValueError(
+ "the values in the last column in mask must be 'True' everywhere."
+ )
# check combination of hist and mask
if not obj.columns.equals(mask.columns):
@@ -439,16 +447,20 @@ class History:
"""
if not isinstance(obj, pd.DataFrame):
- raise TypeError(f"'hist' must be of type pd.DataFrame, but {type(obj).__name__} was given")
+ raise TypeError(
+ f"'hist' must be of type pd.DataFrame, but {type(obj).__name__} was given"
+ )
if any(obj.dtypes != float):
- raise ValueError('dtype of all columns in hist must be float')
+ raise ValueError("dtype of all columns in hist must be float")
if not obj.empty and (
- not obj.columns.equals(pd.Index(range(len(obj.columns))))
- or obj.columns.dtype != int
+ not obj.columns.equals(pd.Index(range(len(obj.columns))))
+ or obj.columns.dtype != int
):
- raise ValueError("column names must be continuous increasing int's, starting with 0.")
+ raise ValueError(
+ "column names must be continuous increasing int's, starting with 0."
+ )
return obj
@@ -458,22 +470,24 @@ class History:
index is not checked !
"""
if not isinstance(obj, pd.Series):
- raise TypeError(f'value must be of type pd.Series, but {type(obj).__name__} was given')
+ raise TypeError(
+ f"value must be of type pd.Series, but {type(obj).__name__} was given"
+ )
if not obj.dtype == float:
- raise ValueError('dtype must be float')
+ raise ValueError("dtype must be float")
return obj
def applyFunctionOnHistory(
- history: History,
- hist_func: callable,
- hist_kws: dict,
- mask_func: callable,
- mask_kws: dict,
- last_column: Union[pd.Series, Literal['dummy'], None] = None,
- func_handle_df: bool = False,
+ history: History,
+ hist_func: callable,
+ hist_kws: dict,
+ mask_func: callable,
+ mask_kws: dict,
+ last_column: Union[pd.Series, Literal["dummy"], None] = None,
+ func_handle_df: bool = False,
):
"""
Apply function on each column in history.
@@ -527,7 +541,7 @@ def applyFunctionOnHistory(
if last_column is None:
new_history.mask.iloc[:, -1:] = True
else:
- if isinstance(last_column, str) and last_column == 'dummy':
+ if isinstance(last_column, str) and last_column == "dummy":
last_column = pd.Series(UNTOUCHED, index=new_history.index, dtype=float)
new_history.append(last_column, force=True)
diff --git a/saqc/core/lib.py b/saqc/core/lib.py
index 24fb29633..85b00ec8c 100644
--- a/saqc/core/lib.py
+++ b/saqc/core/lib.py
@@ -29,11 +29,13 @@ class ConfigController(APIController):
expression: Optional[str] = None
def errorMessage(self):
- return f"line: {self.lineno}\nexpression: {self.expression}" + super().errorMessage()
+ return (
+ f"line: {self.lineno}\nexpression: {self.expression}"
+ + super().errorMessage()
+ )
class SaQCFunction:
-
def __init__(self, name, function, *args, **keywords):
self.name = name
self.func = function
@@ -45,9 +47,7 @@ class SaQCFunction:
def bind(self, *args, **keywords):
return SaQCFunction(
- self.name, self.func,
- *(self.args + args),
- **{**self.keywords, **keywords}
+ self.name, self.func, *(self.args + args), **{**self.keywords, **keywords}
)
def __call__(self, data, field, flags, *args, **keywords):
@@ -55,4 +55,6 @@ class SaQCFunction:
return self.func(data, field, flags, *self.args, *args, **keywords)
def errorMessage(self) -> str:
- return f"function: {self.name}\narguments: {self.args}\nkeywords: {self.keywords}"
+ return (
+ f"function: {self.name}\narguments: {self.args}\nkeywords: {self.keywords}"
+ )
diff --git a/saqc/core/modules/base.py b/saqc/core/modules/base.py
index 3cfba5100..0f357c622 100644
--- a/saqc/core/modules/base.py
+++ b/saqc/core/modules/base.py
@@ -5,7 +5,6 @@ from saqc.core.register import FUNC_MAP
class ModuleBase:
-
def __init__(self, obj):
self.obj = obj
diff --git a/saqc/core/modules/breaks.py b/saqc/core/modules/breaks.py
index c16560c41..785b3cdcc 100644
--- a/saqc/core/modules/breaks.py
+++ b/saqc/core/modules/breaks.py
@@ -11,33 +11,28 @@ from saqc.lib.types import FreqString, IntegerWindow, ColumnName
class Breaks(ModuleBase):
-
def flagMissing(
- self,
- field: ColumnName,
- nodata: float = np.nan,
- flag: float = BAD,
- **kwargs
+ self, field: ColumnName, nodata: float = np.nan, flag: float = BAD, **kwargs
) -> SaQC:
return self.defer("flagMissing", locals())
def flagIsolated(
- self,
- field: ColumnName,
- gap_window: FreqString,
- group_window: FreqString,
- flag: float = BAD,
- **kwargs
+ self,
+ field: ColumnName,
+ gap_window: FreqString,
+ group_window: FreqString,
+ flag: float = BAD,
+ **kwargs
) -> SaQC:
return self.defer("flagIsolated", locals())
def flagJumps(
- self,
- field: ColumnName,
- thresh: float,
- winsz: FreqString,
- min_periods: IntegerWindow=1,
- flag: float = BAD,
- **kwargs
+ self,
+ field: ColumnName,
+ thresh: float,
+ winsz: FreqString,
+ min_periods: IntegerWindow = 1,
+ flag: float = BAD,
+ **kwargs
) -> SaQC:
return self.defer("flagJumps", locals())
diff --git a/saqc/core/modules/changepoints.py b/saqc/core/modules/changepoints.py
index c7bd711f1..cf87a1524 100644
--- a/saqc/core/modules/changepoints.py
+++ b/saqc/core/modules/changepoints.py
@@ -13,40 +13,43 @@ from saqc.lib.types import FreqString, IntegerWindow
class ChangePoints(ModuleBase):
-
def flagChangePoints(
- self, field: str,
- stat_func: Callable[[np.ndarray, np.ndarray], float],
- thresh_func: Callable[[np.ndarray, np.ndarray], float],
- bwd_window: FreqString,
- min_periods_bwd: IntegerWindow,
- fwd_window: Optional[FreqString] = None,
- min_periods_fwd: Optional[IntegerWindow] = None,
- closed: Literal["right", "left", "both", "neither"] = "both",
- try_to_jit: bool = True, # TODO rm, not a user decision
- reduce_window: FreqString = None,
- reduce_func: Callable[[np.ndarray, np.ndarray], int] = lambda x, _: x.argmax(),
- flag: float = BAD,
- **kwargs
+ self,
+ field: str,
+ stat_func: Callable[[np.ndarray, np.ndarray], float],
+ thresh_func: Callable[[np.ndarray, np.ndarray], float],
+ bwd_window: FreqString,
+ min_periods_bwd: IntegerWindow,
+ fwd_window: Optional[FreqString] = None,
+ min_periods_fwd: Optional[IntegerWindow] = None,
+ closed: Literal["right", "left", "both", "neither"] = "both",
+ try_to_jit: bool = True, # TODO rm, not a user decision
+ reduce_window: FreqString = None,
+ reduce_func: Callable[[np.ndarray, np.ndarray], int] = lambda x, _: x.argmax(),
+ flag: float = BAD,
+ **kwargs
) -> SaQC:
return self.defer("flagChangePoints", locals())
def assignChangePointCluster(
- self, field: str,
- stat_func: Callable[[np.array, np.array], float],
- thresh_func: Callable[[np.array, np.array], float],
- bwd_window: str,
- min_periods_bwd: int,
- fwd_window: str = None,
- min_periods_fwd: Optional[int] = None,
- closed: Literal["right", "left", "both", "neither"] = "both",
- try_to_jit: bool = True, # TODO: rm, not a user decision
- reduce_window: str = None,
- reduce_func: Callable[[np.ndarray, np.ndarray], float] = lambda x, _: x.argmax(),
- model_by_resids: bool = False,
- flag_changepoints: bool = False,
- assign_cluster: bool = True,
- flag: float = BAD,
- **kwargs
+ self,
+ field: str,
+ stat_func: Callable[[np.array, np.array], float],
+ thresh_func: Callable[[np.array, np.array], float],
+ bwd_window: str,
+ min_periods_bwd: int,
+ fwd_window: str = None,
+ min_periods_fwd: Optional[int] = None,
+ closed: Literal["right", "left", "both", "neither"] = "both",
+ try_to_jit: bool = True, # TODO: rm, not a user decision
+ reduce_window: str = None,
+ reduce_func: Callable[
+ [np.ndarray, np.ndarray], float
+ ] = lambda x, _: x.argmax(),
+ model_by_resids: bool = False,
+ flag_changepoints: bool = False,
+ assign_cluster: bool = True,
+ flag: float = BAD,
+ **kwargs
) -> SaQC:
return self.defer("assignChangePointCluster", locals())
diff --git a/saqc/core/modules/constants.py b/saqc/core/modules/constants.py
index 862023129..075507757 100644
--- a/saqc/core/modules/constants.py
+++ b/saqc/core/modules/constants.py
@@ -8,25 +8,24 @@ from saqc.lib.types import FreqString, ColumnName
class Constants(ModuleBase):
-
def flagByVariance(
- self,
- field: ColumnName,
- window: FreqString = "12h",
- thresh: float = 0.0005,
- max_missing: int = None,
- max_consec_missing: int = None,
- flag: float = BAD,
- **kwargs
+ self,
+ field: ColumnName,
+ window: FreqString = "12h",
+ thresh: float = 0.0005,
+ max_missing: int = None,
+ max_consec_missing: int = None,
+ flag: float = BAD,
+ **kwargs
) -> SaQC:
return self.defer("flagByVariance", locals())
def flagConstants(
- self,
- field: ColumnName,
- thresh: float,
- window: FreqString,
- flag: float = BAD,
- **kwargs
+ self,
+ field: ColumnName,
+ thresh: float,
+ window: FreqString,
+ flag: float = BAD,
+ **kwargs
) -> SaQC:
return self.defer("flagConstants", locals())
diff --git a/saqc/core/modules/curvefit.py b/saqc/core/modules/curvefit.py
index 4c691bd4a..9557c1d60 100644
--- a/saqc/core/modules/curvefit.py
+++ b/saqc/core/modules/curvefit.py
@@ -12,15 +12,15 @@ from saqc.core.modules.base import ModuleBase
class Curvefit(ModuleBase):
def fitPolynomial(
- self,
- field: str,
- winsz: Union[int, str],
- polydeg: int,
- numba: Literal[True, False, "auto"] = "auto",
- eval_flags: bool = True,
- min_periods: int = 0,
- return_residues: bool = False,
- flag: float = BAD,
- **kwargs
+ self,
+ field: str,
+ winsz: Union[int, str],
+ polydeg: int,
+ numba: Literal[True, False, "auto"] = "auto",
+ eval_flags: bool = True,
+ min_periods: int = 0,
+ return_residues: bool = False,
+ flag: float = BAD,
+ **kwargs
) -> SaQC:
return self.defer("fitPolynomial", locals())
diff --git a/saqc/core/modules/drift.py b/saqc/core/modules/drift.py
index 182cacb18..951219c1d 100644
--- a/saqc/core/modules/drift.py
+++ b/saqc/core/modules/drift.py
@@ -15,76 +15,85 @@ from saqc.lib.types import ColumnName, FreqString, CurveFitter
class Drift(ModuleBase):
def flagDriftFromNorm(
- self,
- field: ColumnName,
- fields: Sequence[ColumnName],
- segment_freq: FreqString,
- norm_spread: float,
- norm_frac: float = 0.5,
- metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
- linkage_method: LinkageString = "single",
- flag: float = BAD,
- **kwargs
+ self,
+ field: ColumnName,
+ fields: Sequence[ColumnName],
+ segment_freq: FreqString,
+ norm_spread: float,
+ norm_frac: float = 0.5,
+ metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(
+ np.array([x, y]), metric="cityblock"
+ )
+ / len(x),
+ linkage_method: LinkageString = "single",
+ flag: float = BAD,
+ **kwargs
) -> SaQC:
return self.defer("flagDriftFromNorm", locals())
def flagDriftFromReference(
- self,
- field: ColumnName,
- fields: Sequence[ColumnName],
- segment_freq: FreqString,
- thresh: float,
- metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
- flag: float = BAD,
- **kwargs
+ self,
+ field: ColumnName,
+ fields: Sequence[ColumnName],
+ segment_freq: FreqString,
+ thresh: float,
+ metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(
+ np.array([x, y]), metric="cityblock"
+ )
+ / len(x),
+ flag: float = BAD,
+ **kwargs
) -> SaQC:
return self.defer("flagDriftFromReference", locals())
def flagDriftFromScaledNorm(
- self,
- field: ColumnName,
- fields_scale1: Sequence[ColumnName],
- fields_scale2: Sequence[ColumnName],
- segment_freq: FreqString,
- norm_spread: float,
- norm_frac: float = 0.5,
- metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
- linkage_method: LinkageString = "single",
- flag: float = BAD,
- **kwargs
+ self,
+ field: ColumnName,
+ fields_scale1: Sequence[ColumnName],
+ fields_scale2: Sequence[ColumnName],
+ segment_freq: FreqString,
+ norm_spread: float,
+ norm_frac: float = 0.5,
+ metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(
+ np.array([x, y]), metric="cityblock"
+ )
+ / len(x),
+ linkage_method: LinkageString = "single",
+ flag: float = BAD,
+ **kwargs
) -> SaQC:
return self.defer("flagDriftFromScaledNorm", locals())
def correctDrift(
- self,
- field: ColumnName,
- maint_data_field: ColumnName,
- cal_mean: int = 5,
- flag_maint_period: bool = False,
- flag: float = BAD,
- **kwargs
+ self,
+ field: ColumnName,
+ maint_data_field: ColumnName,
+ cal_mean: int = 5,
+ flag_maint_period: bool = False,
+ flag: float = BAD,
+ **kwargs
) -> SaQC:
return self.defer("correctDrift", locals())
def correctRegimeAnomaly(
- self,
- field: ColumnName,
- cluster_field: ColumnName,
- model: CurveFitter,
- regime_transmission: Optional[FreqString] = None,
- x_date: bool = False,
- **kwargs
+ self,
+ field: ColumnName,
+ cluster_field: ColumnName,
+ model: CurveFitter,
+ regime_transmission: Optional[FreqString] = None,
+ x_date: bool = False,
+ **kwargs
) -> SaQC:
return self.defer("correctRegimeAnomaly", locals())
def correctOffset(
- self,
- field: ColumnName,
- max_mean_jump: float,
- normal_spread: float,
- search_winsz: FreqString,
- min_periods: int,
- regime_transmission: Optional[FreqString] = None,
- **kwargs
+ self,
+ field: ColumnName,
+ max_mean_jump: float,
+ normal_spread: float,
+ search_winsz: FreqString,
+ min_periods: int,
+ regime_transmission: Optional[FreqString] = None,
+ **kwargs
) -> SaQC:
return self.defer("correctOffset", locals())
diff --git a/saqc/core/modules/flagtools.py b/saqc/core/modules/flagtools.py
index 2d9ea63af..ce221bb93 100644
--- a/saqc/core/modules/flagtools.py
+++ b/saqc/core/modules/flagtools.py
@@ -14,29 +14,25 @@ from saqc.lib.types import ColumnName
class FlagTools(ModuleBase):
-
def clearFlags(self, field: ColumnName, **kwargs) -> SaQC:
return self.defer("clearFlags", locals())
- def forceFlags(
- self, field: ColumnName, flag: float = BAD, **kwargs
- ) -> SaQC:
+ def forceFlags(self, field: ColumnName, flag: float = BAD, **kwargs) -> SaQC:
return self.defer("forceFlags", locals())
- def flagDummy(self, field: ColumnName, **kwargs) -> SaQC:
+ def flagDummy(self, field: ColumnName, **kwargs) -> SaQC:
return self.defer("flagDummy", locals())
- def flagUnflagged(
- self, field: ColumnName, flag: float = BAD, **kwargs
- ) -> SaQC:
+ def flagUnflagged(self, field: ColumnName, flag: float = BAD, **kwargs) -> SaQC:
return self.defer("flagUnflagged", locals())
def flagManual(
- self, field: ColumnName,
- mdata: Union[pd.Series, pd.DataFrame, DictOfSeries],
- mflag: Any = 1,
- method: Literal["plain", "ontime", "left-open", "right-open"] = 'plain',
- flag: float = BAD,
- **kwargs
+ self,
+ field: ColumnName,
+ mdata: Union[pd.Series, pd.DataFrame, DictOfSeries],
+ mflag: Any = 1,
+ method: Literal["plain", "ontime", "left-open", "right-open"] = "plain",
+ flag: float = BAD,
+ **kwargs
) -> SaQC:
return self.defer("flagManual", locals())
diff --git a/saqc/core/modules/generic.py b/saqc/core/modules/generic.py
index 79a37bb09..79d94b1db 100644
--- a/saqc/core/modules/generic.py
+++ b/saqc/core/modules/generic.py
@@ -12,22 +12,21 @@ from saqc.core.modules.base import ModuleBase
class Generic(ModuleBase):
-
def process(
- self,
- field: str,
- func: Callable[[pd.Series], pd.Series],
- nodata: float = np.nan,
- **kwargs
+ self,
+ field: str,
+ func: Callable[[pd.Series], pd.Series],
+ nodata: float = np.nan,
+ **kwargs
) -> SaQC:
return self.defer("process", locals())
def flag(
- self,
- field: str,
- func: Callable[[pd.Series], pd.Series],
- nodata: float = np.nan,
- flag: float = BAD,
- **kwargs
+ self,
+ field: str,
+ func: Callable[[pd.Series], pd.Series],
+ nodata: float = np.nan,
+ flag: float = BAD,
+ **kwargs
) -> SaQC:
return self.defer("flag", locals())
diff --git a/saqc/core/modules/interpolation.py b/saqc/core/modules/interpolation.py
index e617e1cf6..2dd4c44b1 100644
--- a/saqc/core/modules/interpolation.py
+++ b/saqc/core/modules/interpolation.py
@@ -13,38 +13,38 @@ from saqc.funcs.interpolation import _SUPPORTED_METHODS
class Interpolation(ModuleBase):
-
def interpolateByRolling(
- self, field: str,
- winsz: Union[str, int],
- func: Callable[[pd.Series], float] = np.median,
- center: bool = True,
- min_periods: int = 0,
- flag: float = UNFLAGGED,
- **kwargs
+ self,
+ field: str,
+ winsz: Union[str, int],
+ func: Callable[[pd.Series], float] = np.median,
+ center: bool = True,
+ min_periods: int = 0,
+ flag: float = UNFLAGGED,
+ **kwargs
) -> SaQC:
return self.defer("interpolateByRolling", locals())
def interpolateInvalid(
- self,
- field: str,
- method: _SUPPORTED_METHODS,
- inter_order: int = 2,
- inter_limit: int = 2,
- downgrade_interpolation: bool = False,
- flag: float = UNFLAGGED,
- **kwargs
+ self,
+ field: str,
+ method: _SUPPORTED_METHODS,
+ inter_order: int = 2,
+ inter_limit: int = 2,
+ downgrade_interpolation: bool = False,
+ flag: float = UNFLAGGED,
+ **kwargs
) -> SaQC:
return self.defer("interpolateInvalid", locals())
def interpolateIndex(
- self,
- field: str,
- freq: str,
- method: _SUPPORTED_METHODS,
- inter_order: int = 2,
- inter_limit: int = 2,
- downgrade_interpolation: bool = False,
- **kwargs
+ self,
+ field: str,
+ freq: str,
+ method: _SUPPORTED_METHODS,
+ inter_order: int = 2,
+ inter_limit: int = 2,
+ downgrade_interpolation: bool = False,
+ **kwargs
) -> SaQC:
return self.defer("interpolateIndex", locals())
diff --git a/saqc/core/modules/outliers.py b/saqc/core/modules/outliers.py
index 2e4741963..5b9e761f4 100644
--- a/saqc/core/modules/outliers.py
+++ b/saqc/core/modules/outliers.py
@@ -14,108 +14,107 @@ from saqc.lib.types import IntegerWindow, FreqString, ColumnName
class Outliers(ModuleBase):
-
def flagByStray(
- self,
- field: ColumnName,
- partition_freq: Optional[Union[IntegerWindow, FreqString]] = None,
- partition_min: int = 11,
- iter_start: float = 0.5,
- alpha: float = 0.05,
- flag: float = BAD,
- **kwargs
+ self,
+ field: ColumnName,
+ partition_freq: Optional[Union[IntegerWindow, FreqString]] = None,
+ partition_min: int = 11,
+ iter_start: float = 0.5,
+ alpha: float = 0.05,
+ flag: float = BAD,
+ **kwargs,
) -> SaQC:
return self.defer("flagByStray", locals())
def flagMVScores(
- self,
- field: ColumnName,
- fields: Sequence[ColumnName],
- trafo: Callable[[pd.Series], pd.Series] = lambda x: x,
- alpha: float = 0.05,
- n_neighbors: int = 10,
- scoring_func: Callable[[pd.Series], float] = np.sum,
- iter_start: float = 0.5,
- stray_partition: Optional[Union[IntegerWindow, FreqString]] = None,
- stray_partition_min: int = 11,
- trafo_on_partition: bool = True,
- reduction_range: Optional[FreqString] = None,
- reduction_drop_flagged: bool = False, # TODO: still a case ?
- reduction_thresh: float = 3.5,
- reduction_min_periods: int = 1,
- flag: float = BAD,
- **kwargs,
+ self,
+ field: ColumnName,
+ fields: Sequence[ColumnName],
+ trafo: Callable[[pd.Series], pd.Series] = lambda x: x,
+ alpha: float = 0.05,
+ n_neighbors: int = 10,
+ scoring_func: Callable[[pd.Series], float] = np.sum,
+ iter_start: float = 0.5,
+ stray_partition: Optional[Union[IntegerWindow, FreqString]] = None,
+ stray_partition_min: int = 11,
+ trafo_on_partition: bool = True,
+ reduction_range: Optional[FreqString] = None,
+ reduction_drop_flagged: bool = False, # TODO: still a case ?
+ reduction_thresh: float = 3.5,
+ reduction_min_periods: int = 1,
+ flag: float = BAD,
+ **kwargs,
) -> SaQC:
return self.defer("flagMVScores", locals())
def flagRaise(
- self,
- field: ColumnName,
- thresh: float,
- raise_window: FreqString,
- intended_freq: FreqString,
- average_window: Optional[FreqString] = None,
- mean_raise_factor: float = 2.,
- min_slope: Optional[float] = None,
- min_slope_weight: float = 0.8,
- numba_boost: bool = True, # TODO: rm, not a user decision
- flag: float = BAD,
- **kwargs,
+ self,
+ field: ColumnName,
+ thresh: float,
+ raise_window: FreqString,
+ intended_freq: FreqString,
+ average_window: Optional[FreqString] = None,
+ mean_raise_factor: float = 2.0,
+ min_slope: Optional[float] = None,
+ min_slope_weight: float = 0.8,
+ numba_boost: bool = True, # TODO: rm, not a user decision
+ flag: float = BAD,
+ **kwargs,
) -> SaQC:
return self.defer("flagRaise", locals())
def flagMAD(
- self,
- field: ColumnName,
- window: FreqString,
- z: float = 3.5,
- flag: float = BAD,
- **kwargs
+ self,
+ field: ColumnName,
+ window: FreqString,
+ z: float = 3.5,
+ flag: float = BAD,
+ **kwargs,
) -> SaQC:
return self.defer("flagMAD", locals())
def flagOffset(
- self,
- field: ColumnName,
- thresh: float,
- tolerance: float,
- window: Union[IntegerWindow, FreqString],
- rel_thresh: Optional[float] = None,
- numba_kickin: int = 200000, # TODO: rm, not a user decision
- flag: float = BAD,
- **kwargs
+ self,
+ field: ColumnName,
+ thresh: float,
+ tolerance: float,
+ window: Union[IntegerWindow, FreqString],
+ rel_thresh: Optional[float] = None,
+ numba_kickin: int = 200000, # TODO: rm, not a user decision
+ flag: float = BAD,
+ **kwargs,
) -> SaQC:
return self.defer("flagOffset", locals())
def flagByGrubbs(
- self,
- field: ColumnName,
- winsz: Union[FreqString, IntegerWindow],
- alpha: float = 0.05,
- min_periods: int = 8,
- check_lagged: bool = False,
- flag: float = BAD,
- **kwargs
+ self,
+ field: ColumnName,
+ winsz: Union[FreqString, IntegerWindow],
+ alpha: float = 0.05,
+ min_periods: int = 8,
+ check_lagged: bool = False,
+ flag: float = BAD,
+ **kwargs,
) -> SaQC:
return self.defer("flagByGrubbs", locals())
def flagRange(
- self,
- field: ColumnName,
- min: float = -np.inf,
- max: float = np.inf,
- flag: float = BAD,
- **kwargs
+ self,
+ field: ColumnName,
+ min: float = -np.inf,
+ max: float = np.inf,
+ flag: float = BAD,
+ **kwargs,
) -> SaQC:
return self.defer("flagRange", locals())
def flagCrossStatistic(
- self,
- field: ColumnName,
- fields: Sequence[ColumnName],
- thresh: float,
- cross_stat: Literal["modZscore", "Zscore"] = "modZscore",
- flag: float = BAD,
- **kwargs
+ self,
+ field: ColumnName,
+ fields: Sequence[ColumnName],
+ thresh: float,
+ cross_stat: Literal["modZscore", "Zscore"] = "modZscore",
+ flag: float = BAD,
+ **kwargs,
) -> SaQC:
return self.defer("flagCrossStatistic", locals())
diff --git a/saqc/core/modules/pattern.py b/saqc/core/modules/pattern.py
index ddcbfadd8..b976a66aa 100644
--- a/saqc/core/modules/pattern.py
+++ b/saqc/core/modules/pattern.py
@@ -9,25 +9,24 @@ from saqc.core.modules.base import ModuleBase
class Pattern(ModuleBase):
-
def flagPatternByDTW(
- self,
- field: str,
- ref_field: str,
- widths: Sequence[int] = (1, 2, 4, 8),
- waveform: str = "mexh",
- flag: float = BAD,
- **kwargs
+ self,
+ field: str,
+ ref_field: str,
+ widths: Sequence[int] = (1, 2, 4, 8),
+ waveform: str = "mexh",
+ flag: float = BAD,
+ **kwargs
) -> SaQC:
return self.defer("flagPatternByDTW", locals())
def flagPatternByWavelet(
- self,
- field: str,
- ref_field: str,
- max_distance: float = 0.03,
- normalize: bool = True,
- flag: float = BAD,
- **kwargs
+ self,
+ field: str,
+ ref_field: str,
+ max_distance: float = 0.03,
+ normalize: bool = True,
+ flag: float = BAD,
+ **kwargs
) -> SaQC:
return self.defer("flagPatternByWavelet", locals())
diff --git a/saqc/core/modules/resampling.py b/saqc/core/modules/resampling.py
index e590f3728..2171611db 100644
--- a/saqc/core/modules/resampling.py
+++ b/saqc/core/modules/resampling.py
@@ -14,84 +14,88 @@ from saqc.funcs.interpolation import _SUPPORTED_METHODS
class Resampling(ModuleBase):
-
def aggregate(
- self,
- field: str,
- freq: str,
- value_func,
- flag_func: Callable[[pd.Series], float] = np.nanmax,
- method: Literal["fagg", "bagg", "nagg"] = "nagg",
- flag: float = BAD,
- **kwargs
+ self,
+ field: str,
+ freq: str,
+ value_func,
+ flag_func: Callable[[pd.Series], float] = np.nanmax,
+ method: Literal["fagg", "bagg", "nagg"] = "nagg",
+ flag: float = BAD,
+ **kwargs,
) -> SaQC:
return self.defer("aggregate", locals())
- def linear(
- self,
- field: str,
- freq: str,
- **kwargs
- ) -> SaQC:
+ def linear(self, field: str, freq: str, **kwargs) -> SaQC:
return self.defer("linear", locals())
def interpolate(
- self,
- field: str,
- freq: str,
- method: _SUPPORTED_METHODS,
- order: int = 1,
- **kwargs,
+ self,
+ field: str,
+ freq: str,
+ method: _SUPPORTED_METHODS,
+ order: int = 1,
+ **kwargs,
) -> SaQC:
return self.defer("interpolate", locals())
def mapToOriginal(
- self,
- field: str,
- method: Literal[
- "inverse_fagg", "inverse_bagg", "inverse_nagg",
- "inverse_fshift", "inverse_bshift", "inverse_nshift",
- "inverse_interpolation"
- ],
- **kwargs
+ self,
+ field: str,
+ method: Literal[
+ "inverse_fagg",
+ "inverse_bagg",
+ "inverse_nagg",
+ "inverse_fshift",
+ "inverse_bshift",
+ "inverse_nshift",
+ "inverse_interpolation",
+ ],
+ **kwargs,
) -> SaQC:
return self.defer("mapToOriginal", locals())
def shift(
- self,
- field: str,
- freq: str,
- method: Literal["fshift", "bshift", "nshift"] = "nshift",
- freq_check: Optional[Literal["check", "auto"]] = None, # TODO: not a user decision
- **kwargs
+ self,
+ field: str,
+ freq: str,
+ method: Literal["fshift", "bshift", "nshift"] = "nshift",
+ freq_check: Optional[
+ Literal["check", "auto"]
+ ] = None, # TODO: not a user decision
+ **kwargs,
) -> SaQC:
return self.defer("shift", locals())
def resample(
- self,
- field: str,
- freq: str,
- agg_func: Callable[[pd.Series], pd.Series] = np.mean,
- method: Literal["fagg", "bagg", "nagg"] = "bagg",
- max_invalid_total_d: Optional[int] = None,
- max_invalid_consec_d: Optional[int] = None,
- max_invalid_consec_f: Optional[int] = None,
- max_invalid_total_f: Optional[int] = None,
- flag_agg_func: Callable[[pd.Series], float] = max,
- freq_check: Optional[Literal["check", "auto"]] = None,
- **kwargs
+ self,
+ field: str,
+ freq: str,
+ agg_func: Callable[[pd.Series], pd.Series] = np.mean,
+ method: Literal["fagg", "bagg", "nagg"] = "bagg",
+ max_invalid_total_d: Optional[int] = None,
+ max_invalid_consec_d: Optional[int] = None,
+ max_invalid_consec_f: Optional[int] = None,
+ max_invalid_total_f: Optional[int] = None,
+ flag_agg_func: Callable[[pd.Series], float] = max,
+ freq_check: Optional[Literal["check", "auto"]] = None,
+ **kwargs,
) -> SaQC:
return self.defer("resample", locals())
def reindexFlags(
- self,
- field: str,
- method: Literal[
- "inverse_fagg", "inverse_bagg", "inverse_nagg",
- "inverse_fshift", "inverse_bshift", "inverse_nshift"
- ],
- source: str,
- freq: Optional[str] = None,
- **kwargs
+ self,
+ field: str,
+ method: Literal[
+ "inverse_fagg",
+ "inverse_bagg",
+ "inverse_nagg",
+ "inverse_fshift",
+ "inverse_bshift",
+ "inverse_nshift",
+ ],
+ source: str,
+ freq: Optional[str] = None,
+ **kwargs,
) -> SaQC:
return self.defer("reindexFlags", locals())
diff --git a/saqc/core/modules/residues.py b/saqc/core/modules/residues.py
index c66e0165b..c71b145eb 100644
--- a/saqc/core/modules/residues.py
+++ b/saqc/core/modules/residues.py
@@ -12,29 +12,28 @@ from saqc.core.modules.base import ModuleBase
class Residues(ModuleBase):
-
def calculatePolynomialResidues(
- self,
- field: str,
- winsz: Union[str, int],
- polydeg: int,
- numba: Literal[True, False, "auto"] = "auto", # TODO: rm, not a a user decision
- eval_flags: bool = True, # TODO, not valid anymore, if still needed, maybe assign user-passed ``flag``?
- min_periods: Optional[int] = 0,
- flag: float = BAD,
- **kwargs
+ self,
+ field: str,
+ winsz: Union[str, int],
+ polydeg: int,
+ numba: Literal[True, False, "auto"] = "auto", # TODO: rm, not a a user decision
+ eval_flags: bool = True, # TODO, not valid anymore, if still needed, maybe assign user-passed ``flag``?
+ min_periods: Optional[int] = 0,
+ flag: float = BAD,
+ **kwargs
) -> SaQC:
return self.defer("calculatePolynomialResidues", locals())
def calculateRollingResidues(
- self,
- field: str,
- winsz: Union[str, int],
- func: Callable[[np.ndarray], np.ndarray] = np.mean,
- eval_flags: bool = True,
- min_periods: Optional[int] = 0,
- center: bool = True,
- flag: float = BAD,
- **kwargs
+ self,
+ field: str,
+ winsz: Union[str, int],
+ func: Callable[[np.ndarray], np.ndarray] = np.mean,
+ eval_flags: bool = True,
+ min_periods: Optional[int] = 0,
+ center: bool = True,
+ flag: float = BAD,
+ **kwargs
) -> SaQC:
return self.defer("calculateRollingResidues", locals())
diff --git a/saqc/core/modules/rolling.py b/saqc/core/modules/rolling.py
index d29cb4018..7d983fe0d 100644
--- a/saqc/core/modules/rolling.py
+++ b/saqc/core/modules/rolling.py
@@ -12,15 +12,15 @@ from saqc.core.modules.base import ModuleBase
class Rolling(ModuleBase):
def roll(
- self,
- field: str,
- winsz: Union[str, int],
- func: Callable[[pd.Series], float]=np.mean,
- eval_flags: bool=True, # TODO: not applicable anymore
- min_periods: int=0,
- center: bool=True,
- return_residues=False, # TODO: this should not be public, a wrapper would be better
- flag: float = BAD,
- **kwargs
+ self,
+ field: str,
+ winsz: Union[str, int],
+ func: Callable[[pd.Series], float] = np.mean,
+ eval_flags: bool = True, # TODO: not applicable anymore
+ min_periods: int = 0,
+ center: bool = True,
+ return_residues=False, # TODO: this should not be public, a wrapper would be better
+ flag: float = BAD,
+ **kwargs
):
return self.defer("roll", locals())
diff --git a/saqc/core/modules/scores.py b/saqc/core/modules/scores.py
index 261566da4..6fa7cfb64 100644
--- a/saqc/core/modules/scores.py
+++ b/saqc/core/modules/scores.py
@@ -13,21 +13,20 @@ from saqc.core.modules.base import ModuleBase
class Scores(ModuleBase):
-
def assignKNNScore(
- self,
- field: str,
- fields: Sequence[str],
- n_neighbors: int = 10,
- trafo: Callable[[pd.Series], pd.Series] = lambda x: x,
- trafo_on_partition: bool = True,
- scoring_func: Callable[[pd.Series], float] = np.sum,
- target_field: str = 'kNN_scores',
- partition_freq: Union[float, str] = np.inf,
- partition_min: int = 2,
- kNN_algorithm: Literal["ball_tree", "kd_tree", "brute", "auto"] = 'ball_tree',
- metric: str = 'minkowski',
- p: int = 2,
- **kwargs
+ self,
+ field: str,
+ fields: Sequence[str],
+ n_neighbors: int = 10,
+ trafo: Callable[[pd.Series], pd.Series] = lambda x: x,
+ trafo_on_partition: bool = True,
+ scoring_func: Callable[[pd.Series], float] = np.sum,
+ target_field: str = "kNN_scores",
+ partition_freq: Union[float, str] = np.inf,
+ partition_min: int = 2,
+ kNN_algorithm: Literal["ball_tree", "kd_tree", "brute", "auto"] = "ball_tree",
+ metric: str = "minkowski",
+ p: int = 2,
+ **kwargs
) -> SaQC:
return self.defer("assignKNNScore", locals())
diff --git a/saqc/core/modules/tools.py b/saqc/core/modules/tools.py
index 6a506d258..cce269572 100644
--- a/saqc/core/modules/tools.py
+++ b/saqc/core/modules/tools.py
@@ -22,13 +22,13 @@ class Tools(ModuleBase):
return self.defer("rename", locals())
def mask(
- self,
- field: str,
- mode: Literal["periodic", "mask_var"],
- mask_var: Optional[str]=None,
- period_start: Optional[str]=None,
- period_end: Optional[str]=None,
- include_bounds: bool=True,
- **kwargs,
+ self,
+ field: str,
+ mode: Literal["periodic", "mask_var"],
+ mask_var: Optional[str] = None,
+ period_start: Optional[str] = None,
+ period_end: Optional[str] = None,
+ include_bounds: bool = True,
+ **kwargs,
) -> SaQC:
return self.defer("mask", locals())
diff --git a/saqc/core/modules/transformation.py b/saqc/core/modules/transformation.py
index 036565d03..749d2b444 100644
--- a/saqc/core/modules/transformation.py
+++ b/saqc/core/modules/transformation.py
@@ -10,12 +10,11 @@ from saqc.core.modules.base import ModuleBase
class Transformation(ModuleBase):
-
def transform(
- self,
- field: str,
- func: Callable[[pd.Series], pd.Series],
- partition_freq: Optional[Union[float, str]] = None,
- **kwargs
+ self,
+ field: str,
+ func: Callable[[pd.Series], pd.Series],
+ partition_freq: Optional[Union[float, str]] = None,
+ **kwargs
) -> SaQC:
return self.defer("transform", locals())
diff --git a/saqc/core/reader.py b/saqc/core/reader.py
index ce905f762..d13ec81a2 100644
--- a/saqc/core/reader.py
+++ b/saqc/core/reader.py
@@ -23,7 +23,7 @@ def _handleEmptyLines(df):
df = df.reset_index()
i = (df == F.VARNAME).first_valid_index()
df.columns = df.iloc[i]
- df = df.iloc[i + 1:]
+ df = df.iloc[i + 1 :]
# mark empty lines
mask = (df.isnull() | (df == "")).all(axis=1)
@@ -77,11 +77,7 @@ def _parseConfig(df, flags, nodata):
regex=regex,
)
- control = ConfigController(
- plot=plot,
- lineno=lineno + 2,
- expression=expr
- )
+ control = ConfigController(plot=plot, lineno=lineno + 2, expression=expr)
f = func.bind(**{"nodata": nodata, **kwargs})
diff --git a/saqc/core/register.py b/saqc/core/register.py
index f0d56b58e..bad96aba5 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -61,7 +61,9 @@ def register(masking: MaskingStrT = "all", module: Optional[str] = None):
return inner
-def _preCall(func: callable, args: tuple, kwargs: dict, masking: MaskingStrT, fname: str):
+def _preCall(
+ func: callable, args: tuple, kwargs: dict, masking: MaskingStrT, fname: str
+):
"""
Handler that runs before any call to a saqc-function.
@@ -97,7 +99,7 @@ def _preCall(func: callable, args: tuple, kwargs: dict, masking: MaskingStrT, fn
"""
mthresh = _getMaskingThresh(masking, kwargs, fname)
- kwargs['to_mask'] = mthresh
+ kwargs["to_mask"] = mthresh
data, field, flags, *args = args
@@ -108,10 +110,14 @@ def _preCall(func: callable, args: tuple, kwargs: dict, masking: MaskingStrT, fn
# store current state
state = CallState(
func=func,
- data=data, flags=flags, field=field,
- args=args, kwargs=kwargs,
- masking=masking, mthresh=mthresh,
- mask=mask
+ data=data,
+ flags=flags,
+ field=field,
+ args=args,
+ kwargs=kwargs,
+ masking=masking,
+ mthresh=mthresh,
+ mask=mask,
)
# handle flags - clearing
@@ -157,11 +163,11 @@ def _getMaskingColumns(data: dios.DictOfSeries, field: str, masking: MaskingStrT
------
ValueError: if given masking literal is not supported
"""
- if masking == 'all':
+ if masking == "all":
return data.columns
- if masking == 'none':
+ if masking == "none":
return pd.Index([])
- if masking == 'field':
+ if masking == "field":
return pd.Index([field])
raise ValueError(f"wrong use of `register(masking={masking})`")
@@ -195,18 +201,20 @@ def _getMaskingThresh(masking, kwargs, fname):
- ``+np.inf``, if ``False``
If a floatish ``to_mask`` is found in the kwargs, this value is taken as the threshold.
"""
- if 'to_mask' not in kwargs:
+ if "to_mask" not in kwargs:
return UNFLAGGED
- thresh = kwargs['to_mask']
+ thresh = kwargs["to_mask"]
if not isinstance(thresh, (bool, float, int)):
raise TypeError(f"'to_mask' must be of type bool or float")
- if masking == 'none' and thresh not in (False, np.inf):
+ if masking == "none" and thresh not in (False, np.inf):
# TODO: fix warning reference to docu
- warnings.warn(f"the saqc-function {fname!r} ignores masking and therefore does not evaluate the passed "
- f"'to_mask'-keyword. Please refer to the documentation: TODO")
+ warnings.warn(
+ f"the saqc-function {fname!r} ignores masking and therefore does not evaluate the passed "
+ f"'to_mask'-keyword. Please refer to the documentation: TODO"
+ )
if thresh is True: # masking ON
thresh = UNFLAGGED
@@ -220,7 +228,9 @@ def _getMaskingThresh(masking, kwargs, fname):
# TODO: this is heavily undertested
-def _maskData(data, flags, columns, thresh) -> Tuple[dios.DictOfSeries, dios.DictOfSeries]:
+def _maskData(
+ data, flags, columns, thresh
+) -> Tuple[dios.DictOfSeries, dios.DictOfSeries]:
"""
Mask data with Nans by flags worse that a threshold and according to ``masking`` keyword
from the functions decorator.
@@ -249,7 +259,9 @@ def _maskData(data, flags, columns, thresh) -> Tuple[dios.DictOfSeries, dios.Dic
return data, mask
-def _isflagged(flagscol: Union[np.array, pd.Series], thresh: float) -> Union[np.array, pd.Series]:
+def _isflagged(
+ flagscol: Union[np.array, pd.Series], thresh: float
+) -> Union[np.array, pd.Series]:
"""
Return a mask of flags accordingly to `thresh`. Return type is same as flags.
"""
@@ -268,19 +280,19 @@ def _prepareFlags(flags: Flags, masking) -> Flags:
the saqc-function needs.
"""
# Either the index or the columns itself changed
- if masking == 'none':
+ if masking == "none":
return flags.copy()
return initFlagsLike(flags, initial_value=UNTOUCHED)
def _restoreFlags(flags: Flags, old_state: CallState):
- if old_state.masking == 'none':
+ if old_state.masking == "none":
return flags
columns = flags.columns
# take field column and all possibly newly added columns
- if old_state.masking == 'field':
+ if old_state.masking == "field":
columns = columns.difference(old_state.flags.columns)
columns = columns.append(pd.Index([old_state.field]))
@@ -303,7 +315,7 @@ def _unmaskData(data: dios.DictOfSeries, old_state: CallState) -> dios.DictOfSer
-----
Even if this returns data, it work inplace !
"""
- if old_state.masking == 'none':
+ if old_state.masking == "none":
return data
# we have two options to implement this:
@@ -327,7 +339,9 @@ def _unmaskData(data: dios.DictOfSeries, old_state: CallState) -> dios.DictOfSer
# col in new only : new (keep column)
# col in old only : new (ignore, was deleted)
- columns = old_state.mask.columns.intersection(data.columns) # in old, in masked, in new
+ columns = old_state.mask.columns.intersection(
+ data.columns
+ ) # in old, in masked, in new
for c in columns:
@@ -350,4 +364,3 @@ def _unmaskData(data: dios.DictOfSeries, old_state: CallState) -> dios.DictOfSer
data.loc[:, c] = np.where(restore_old_mask, old, new)
return data
-
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
index b8418aafc..771d30b9f 100644
--- a/saqc/core/translator.py
+++ b/saqc/core/translator.py
@@ -20,6 +20,7 @@ from saqc.lib.types import ExternalFlag, MaterializedGraph
ForwardMap = Dict[ExternalFlag, float]
BackwardMap = Dict[float, ExternalFlag]
+
class Translator:
"""
This class provides the basic translation mechanism and should serve as
@@ -46,7 +47,7 @@ class Translator:
TO_MASK = True
- def __init__(self, forward: ForwardMap, backward: Optional[BackwardMap]=None):
+ def __init__(self, forward: ForwardMap, backward: Optional[BackwardMap] = None):
"""
Parameters
----------
@@ -67,11 +68,15 @@ class Translator:
if backward is None:
backward = {v: k for k, v in forward.items()}
if {UNFLAGGED, BAD} - set(backward.keys()):
- raise ValueError(f"need translations for the special flags `UNFLAGGED` ({UNFLAGGED}) and `BAD` ({BAD})")
+ raise ValueError(
+ f"need translations for the special flags `UNFLAGGED` ({UNFLAGGED}) and `BAD` ({BAD})"
+ )
self._backward = backward
@staticmethod
- def _translate(flags: Union[Flags, pd.DataFrame], trans_map: Union[ForwardMap, BackwardMap]) -> DictOfSeries:
+ def _translate(
+ flags: Union[Flags, pd.DataFrame], trans_map: Union[ForwardMap, BackwardMap]
+ ) -> DictOfSeries:
"""
Translate a given flag data structure to another one according to the
mapping given in `trans_map`
@@ -158,7 +163,10 @@ class FloatTranslator(Translator):
internal float flags
"""
- _FORWARD: Dict[float, float] = {-np.inf: -np.inf, **{k: k for k in np.arange(0, 256, dtype=float)}}
+ _FORWARD: Dict[float, float] = {
+ -np.inf: -np.inf,
+ **{k: k for k in np.arange(0, 256, dtype=float)},
+ }
def __init__(self):
super().__init__(self._FORWARD)
@@ -171,8 +179,17 @@ class DmpTranslator(Translator):
the UFZ - Datamanagementportal
"""
- _FORWARD: Dict[str, float] = {"NIL": UNFLAGGED, "OK": GOOD, "DOUBTFUL": DOUBTFUL, "BAD": BAD}
- _COL_LABELS: Dict[str, str] = {"flag": "quality_flag", "comment": "quality_comment", "cause": "quality_cause"}
+ _FORWARD: Dict[str, float] = {
+ "NIL": UNFLAGGED,
+ "OK": GOOD,
+ "DOUBTFUL": DOUBTFUL,
+ "BAD": BAD,
+ }
+ _COL_LABELS: Dict[str, str] = {
+ "flag": "quality_flag",
+ "comment": "quality_comment",
+ "cause": "quality_cause",
+ }
def __init__(self):
super().__init__(forward=self._FORWARD)
@@ -213,7 +230,9 @@ class DmpTranslator(Translator):
if not isinstance(cols, pd.MultiIndex):
raise TypeError("DMP-Flags need mult-index columns")
if set(cols.get_level_values(1)) != set(self._COL_LABELS.values()):
- raise TypeError(f"DMP-Flags expect the labels 'list(self._COL_LABELS.values)' in the secondary level")
+ raise TypeError(
+ f"DMP-Flags expect the labels 'list(self._COL_LABELS.values)' in the secondary level"
+ )
qflags = flags.xs(key=self._COL_LABELS["flag"], axis="columns", level=1)
return super().forward(qflags) # type: ignore
@@ -243,11 +262,15 @@ class DmpTranslator(Translator):
# we prepend empty strings to handle default columns in `Flags`
# and potentially given flags not generated during the saqc run,
# represented by `call_stack`
- flag_funcs = ([""] * (len(flag_history.hist.columns) - len(flag_funcs))) + flag_funcs
+ flag_funcs = (
+ [""] * (len(flag_history.hist.columns) - len(flag_funcs))
+ ) + flag_funcs
var_flags = {
- self._COL_LABELS["flag"] : tflags[field],
- self._COL_LABELS["comment"] : flag_pos.apply(lambda p: json.dumps({"test": flag_funcs[p]})),
- self._COL_LABELS["cause"] : "",
+ self._COL_LABELS["flag"]: tflags[field],
+ self._COL_LABELS["comment"]: flag_pos.apply(
+ lambda p: json.dumps({"test": flag_funcs[p]})
+ ),
+ self._COL_LABELS["cause"]: "",
}
out[field] = pd.DataFrame(var_flags)
return pd.concat(out, axis="columns")
@@ -260,7 +283,13 @@ class PositionalTranslator(Translator):
"""
_FORWARD: Dict[int, float] = {0: UNFLAGGED, 1: DOUBTFUL, 2: BAD}
- _BACKWARD: Dict[float, int] = {UNTOUCHED: 0, UNFLAGGED: 0, GOOD: 0, DOUBTFUL: 1, BAD: 2}
+ _BACKWARD: Dict[float, int] = {
+ UNTOUCHED: 0,
+ UNFLAGGED: 0,
+ GOOD: 0,
+ DOUBTFUL: 1,
+ BAD: 2,
+ }
def __init__(self):
super().__init__(forward=self._FORWARD, backward=self._BACKWARD)
@@ -282,12 +311,13 @@ class PositionalTranslator(Translator):
for field in flags.columns:
# drop the first column (i.e. the '9')
fflags = pd.DataFrame(
- flags[field].apply(tuple).tolist(),
- index=flags[field].index
+ flags[field].apply(tuple).tolist(), index=flags[field].index
).iloc[:, 1:]
tflags = super().forward(fflags.astype(int)).toFrame()
- tflags.insert(loc=0, column=0, value=pd.Series(UNFLAGGED, index=fflags.index))
+ tflags.insert(
+ loc=0, column=0, value=pd.Series(UNFLAGGED, index=fflags.index)
+ )
data[field] = tflags
return Flags(data)
@@ -310,9 +340,9 @@ class PositionalTranslator(Translator):
out = {}
for field in flags.columns:
thist = flags.history[field].hist.replace(self._BACKWARD)
- tflags = (thist
- .astype(int).astype(str)
- .apply(lambda x: x.sum(), axis="columns"))
+ tflags = (
+ thist.astype(int).astype(str).apply(lambda x: x.sum(), axis="columns")
+ )
# NOTE: work around the default first column history columns (see GL#182)
out[field] = "9" + tflags.str.slice(start=1)
return pd.DataFrame(out)
diff --git a/saqc/core/visitor.py b/saqc/core/visitor.py
index 7d7203fa0..d301d3108 100644
--- a/saqc/core/visitor.py
+++ b/saqc/core/visitor.py
@@ -134,7 +134,7 @@ class ConfigFunctionParser(ast.NodeVisitor):
ast.Index,
ast.USub,
ast.List,
- ast.Attribute
+ ast.Attribute,
)
def __init__(self, flags):
@@ -207,7 +207,7 @@ class ConfigFunctionParser(ast.NodeVisitor):
co = compile(
ast.fix_missing_locations(ast.Interactive(body=[vnode])),
"<ast>",
- mode="single"
+ mode="single",
)
# NOTE: only pass a copy to not clutter the self.environment
exec(co, {**self.environment}, self.kwargs)
diff --git a/saqc/funcs/breaks.py b/saqc/funcs/breaks.py
index b6dd00834..ff6af9e8b 100644
--- a/saqc/funcs/breaks.py
+++ b/saqc/funcs/breaks.py
@@ -23,14 +23,14 @@ from saqc.funcs.changepoints import assignChangePointCluster
from saqc.core import register, Flags
-@register(masking='field', module="breaks")
+@register(masking="field", module="breaks")
def flagMissing(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- nodata: float = np.nan,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ nodata: float = np.nan,
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
The function flags all values indicating missing data.
@@ -65,15 +65,15 @@ def flagMissing(
return data, flags
-@register(masking='field', module="breaks")
+@register(masking="field", module="breaks")
def flagIsolated(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- gap_window: FreqString,
- group_window: FreqString,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ gap_window: FreqString,
+ group_window: FreqString,
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
The function flags arbitrary large groups of values, if they are surrounded by sufficiently
@@ -129,9 +129,9 @@ def flagIsolated(
start = srs.index[0]
stop = srs.index[-1]
if stop - start <= group_window:
- left = mask[start - gap_window: start].iloc[:-1]
+ left = mask[start - gap_window : start].iloc[:-1]
if left.all():
- right = mask[stop: stop + gap_window].iloc[1:]
+ right = mask[stop : stop + gap_window].iloc[1:]
if right.all():
bools[start:stop] = True
@@ -139,16 +139,16 @@ def flagIsolated(
return data, flags
-@register(masking='field', module="breaks")
+@register(masking="field", module="breaks")
def flagJumps(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- thresh: float,
- winsz: FreqString,
- min_periods: IntegerWindow = 1,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ thresh: float,
+ winsz: FreqString,
+ min_periods: IntegerWindow = 1,
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
Flag datapoints, where the mean of the values significantly changes (where the value course "jumps").
@@ -173,7 +173,9 @@ def flagJumps(
flag to set.
"""
return assignChangePointCluster(
- data, field, flags,
+ data,
+ field,
+ flags,
stat_func=lambda x, y: np.abs(np.mean(x) - np.mean(y)),
thresh_func=lambda x, y: thresh,
bwd_window=winsz,
diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py
index 450a77602..7325b6467 100644
--- a/saqc/funcs/changepoints.py
+++ b/saqc/funcs/changepoints.py
@@ -19,21 +19,23 @@ from saqc.lib.types import ColumnName, FreqString, IntegerWindow
logger = logging.getLogger("SaQC")
-@register(masking='field', module="changepoints")
+@register(masking="field", module="changepoints")
def flagChangePoints(
- data: DictOfSeries, field: str, flags: Flags,
- stat_func: Callable[[np.ndarray, np.ndarray], float],
- thresh_func: Callable[[np.ndarray, np.ndarray], float],
- bwd_window: FreqString,
- min_periods_bwd: IntegerWindow,
- fwd_window: Optional[FreqString] = None,
- min_periods_fwd: Optional[IntegerWindow] = None,
- closed: Literal["right", "left", "both", "neither"] = "both",
- try_to_jit: bool = True, # TODO rm, not a user decision
- reduce_window: FreqString = None,
- reduce_func: Callable[[np.ndarray, np.ndarray], int] = lambda x, _: x.argmax(),
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ stat_func: Callable[[np.ndarray, np.ndarray], float],
+ thresh_func: Callable[[np.ndarray, np.ndarray], float],
+ bwd_window: FreqString,
+ min_periods_bwd: IntegerWindow,
+ fwd_window: Optional[FreqString] = None,
+ min_periods_fwd: Optional[IntegerWindow] = None,
+ closed: Literal["right", "left", "both", "neither"] = "both",
+ try_to_jit: bool = True, # TODO rm, not a user decision
+ reduce_window: FreqString = None,
+ reduce_func: Callable[[np.ndarray, np.ndarray], int] = lambda x, _: x.argmax(),
+ flag: float = BAD,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
Flag datapoints, where the parametrization of the process, the data is assumed to generate by, significantly
@@ -103,28 +105,30 @@ def flagChangePoints(
model_by_resids=False,
assign_cluster=False,
flag=flag,
- **kwargs
+ **kwargs,
)
-@register(masking='field', module="changepoints")
+@register(masking="field", module="changepoints")
def assignChangePointCluster(
- data: DictOfSeries, field: str, flags: Flags,
- stat_func: Callable[[np.array, np.array], float],
- thresh_func: Callable[[np.array, np.array], float],
- bwd_window: str,
- min_periods_bwd: int,
- fwd_window: str = None,
- min_periods_fwd: Optional[int] = None,
- closed: Literal["right", "left", "both", "neither"] = "both",
- try_to_jit: bool = True, # TODO: rm, not a user decision
- reduce_window: str = None,
- reduce_func: Callable[[np.ndarray, np.ndarray], float] = lambda x, _: x.argmax(),
- model_by_resids: bool = False,
- flag_changepoints: bool = False,
- assign_cluster: bool = True,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ stat_func: Callable[[np.array, np.array], float],
+ thresh_func: Callable[[np.array, np.array], float],
+ bwd_window: str,
+ min_periods_bwd: int,
+ fwd_window: str = None,
+ min_periods_fwd: Optional[int] = None,
+ closed: Literal["right", "left", "both", "neither"] = "both",
+ try_to_jit: bool = True, # TODO: rm, not a user decision
+ reduce_window: str = None,
+ reduce_func: Callable[[np.ndarray, np.ndarray], float] = lambda x, _: x.argmax(),
+ model_by_resids: bool = False,
+ flag_changepoints: bool = False,
+ assign_cluster: bool = True,
+ flag: float = BAD,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
Assigns label to the data, aiming to reflect continous regimes of the processes the data is assumed to be
@@ -194,12 +198,19 @@ def assignChangePointCluster(
reduce_window = f"{int(pd.Timedelta(bwd_window).total_seconds() + pd.Timedelta(fwd_window).total_seconds())}s"
roller = customRoller(data_ser, window=bwd_window)
- bwd_start, bwd_end = roller.window.get_window_bounds(len(data_ser), min_periods=min_periods_bwd, closed=closed)
+ bwd_start, bwd_end = roller.window.get_window_bounds(
+ len(data_ser), min_periods=min_periods_bwd, closed=closed
+ )
roller = customRoller(data_ser, window=fwd_window, forward=True)
- fwd_start, fwd_end = roller.window.get_window_bounds(len(data_ser), min_periods=min_periods_fwd, closed=closed)
+ fwd_start, fwd_end = roller.window.get_window_bounds(
+ len(data_ser), min_periods=min_periods_fwd, closed=closed
+ )
- min_mask = ~((fwd_end - fwd_start <= min_periods_fwd) | (bwd_end - bwd_start <= min_periods_bwd))
+ min_mask = ~(
+ (fwd_end - fwd_start <= min_periods_fwd)
+ | (bwd_end - bwd_start <= min_periods_bwd)
+ )
fwd_end = fwd_end[min_mask]
split = bwd_end[min_mask]
bwd_start = bwd_start[min_mask]
@@ -211,14 +222,18 @@ def assignChangePointCluster(
jit_sf = numba.jit(stat_func, nopython=True)
jit_tf = numba.jit(thresh_func, nopython=True)
try:
- jit_sf(data_arr[bwd_start[0]:bwd_end[0]], data_arr[fwd_start[0]:fwd_end[0]])
- jit_tf(data_arr[bwd_start[0]:bwd_end[0]], data_arr[fwd_start[0]:fwd_end[0]])
+ jit_sf(
+ data_arr[bwd_start[0] : bwd_end[0]], data_arr[fwd_start[0] : fwd_end[0]]
+ )
+ jit_tf(
+ data_arr[bwd_start[0] : bwd_end[0]], data_arr[fwd_start[0] : fwd_end[0]]
+ )
stat_func = jit_sf
thresh_func = jit_tf
try_to_jit = True
except (numba.TypingError, numba.UnsupportedError, IndexError):
try_to_jit = False
- logging.warning('Could not jit passed statistic - omitting jitting!')
+ logging.warning("Could not jit passed statistic - omitting jitting!")
args = data_arr, bwd_start, fwd_end, split, stat_func, thresh_func, check_len
@@ -241,9 +256,13 @@ def assignChangePointCluster(
if reduce_window:
l = detected.shape[0]
roller = customRoller(detected, window=reduce_window)
- start, end = roller.window.get_window_bounds(num_values=l, min_periods=1, closed='both', center=True)
+ start, end = roller.window.get_window_bounds(
+ num_values=l, min_periods=1, closed="both", center=True
+ )
- detected = _reduceCPCluster(stat_arr[result_arr], thresh_arr[result_arr], start, end, reduce_func, l)
+ detected = _reduceCPCluster(
+ stat_arr[result_arr], thresh_arr[result_arr], start, end, reduce_func, l
+ )
det_index = det_index[detected]
if assign_cluster:
@@ -261,23 +280,27 @@ def assignChangePointCluster(
@numba.jit(parallel=True, nopython=True)
-def _slidingWindowSearchNumba(data_arr, bwd_start, fwd_end, split, stat_func, thresh_func, num_val):
+def _slidingWindowSearchNumba(
+ data_arr, bwd_start, fwd_end, split, stat_func, thresh_func, num_val
+):
stat_arr = np.zeros(num_val)
thresh_arr = np.zeros(num_val)
for win_i in numba.prange(0, num_val - 1):
- x = data_arr[bwd_start[win_i]:split[win_i]]
- y = data_arr[split[win_i]:fwd_end[win_i]]
+ x = data_arr[bwd_start[win_i] : split[win_i]]
+ y = data_arr[split[win_i] : fwd_end[win_i]]
stat_arr[win_i] = stat_func(x, y)
thresh_arr[win_i] = thresh_func(x, y)
return stat_arr, thresh_arr
-def _slidingWindowSearch(data_arr, bwd_start, fwd_end, split, stat_func, thresh_func, num_val):
+def _slidingWindowSearch(
+ data_arr, bwd_start, fwd_end, split, stat_func, thresh_func, num_val
+):
stat_arr = np.zeros(num_val)
thresh_arr = np.zeros(num_val)
for win_i in range(0, num_val - 1):
- x = data_arr[bwd_start[win_i]:split[win_i]]
- y = data_arr[split[win_i]:fwd_end[win_i]]
+ x = data_arr[bwd_start[win_i] : split[win_i]]
+ y = data_arr[split[win_i] : fwd_end[win_i]]
stat_arr[win_i] = stat_func(x, y)
thresh_arr[win_i] = thresh_func(x, y)
return stat_arr, thresh_arr
diff --git a/saqc/funcs/constants.py b/saqc/funcs/constants.py
index 6f8d29828..f4bc49f04 100644
--- a/saqc/funcs/constants.py
+++ b/saqc/funcs/constants.py
@@ -16,15 +16,15 @@ from saqc.lib.tools import customRoller, getFreqDelta
from saqc.lib.types import FreqString, ColumnName
-@register(masking='field', module="constants")
+@register(masking="field", module="constants")
def flagConstants(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- thresh: float,
- window: FreqString,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ thresh: float,
+ window: FreqString,
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
This functions flags plateaus/series of constant values of length `window` if
@@ -61,7 +61,7 @@ def flagConstants(
Flags values may have changed, relatively to the flags input.
"""
if not isinstance(window, str):
- raise TypeError('window must be offset string.')
+ raise TypeError("window must be offset string.")
d = data[field]
# min_periods=2 ensures that at least two non-nan values are present
@@ -80,17 +80,17 @@ def flagConstants(
return data, flags
-@register(masking='field', module="constants")
+@register(masking="field", module="constants")
def flagByVariance(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- window: FreqString = "12h",
- thresh: float = 0.0005,
- max_missing: int = None,
- max_consec_missing: int = None,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ window: FreqString = "12h",
+ thresh: float = 0.0005,
+ max_missing: int = None,
+ max_consec_missing: int = None,
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
Function flags plateaus/series of constant values. Any interval of values y(t),..y(t+n) is flagged, if:
@@ -133,7 +133,7 @@ def flagByVariance(
delta = getFreqDelta(dataseries.index)
if not delta:
- raise IndexError('Timeseries irregularly sampled!')
+ raise IndexError("Timeseries irregularly sampled!")
if max_missing is None:
max_missing = np.inf
diff --git a/saqc/funcs/curvefit.py b/saqc/funcs/curvefit.py
index 4b50693c6..627d9c953 100644
--- a/saqc/funcs/curvefit.py
+++ b/saqc/funcs/curvefit.py
@@ -16,23 +16,23 @@ from saqc.lib.ts_operators import (
polyRollerNumba,
polyRoller,
polyRollerNoMissingNumba,
- polyRollerNoMissing
+ polyRollerNoMissing,
)
-@register(masking='field', module="curvefit")
+@register(masking="field", module="curvefit")
def fitPolynomial(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- winsz: Union[int, str],
- polydeg: int,
- numba: Literal[True, False, "auto"] = "auto",
- eval_flags: bool = True,
- min_periods: int = 0,
- return_residues: bool = False,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ winsz: Union[int, str],
+ polydeg: int,
+ numba: Literal[True, False, "auto"] = "auto",
+ eval_flags: bool = True,
+ min_periods: int = 0,
+ return_residues: bool = False,
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
Function fits a polynomial model to the data and returns the fitted data curve.
@@ -117,25 +117,36 @@ def fitPolynomial(
regular = getFreqDelta(to_fit.index)
if not regular:
if isinstance(winsz, int):
- raise NotImplementedError("Integer based window size is not supported for not-harmonized" "sample series.")
+ raise NotImplementedError(
+ "Integer based window size is not supported for not-harmonized"
+ "sample series."
+ )
# get interval centers
- centers = (to_fit.rolling(pd.Timedelta(winsz) / 2, closed="both", min_periods=min_periods).count()).floor()
+ centers = (
+ to_fit.rolling(
+ pd.Timedelta(winsz) / 2, closed="both", min_periods=min_periods
+ ).count()
+ ).floor()
centers = centers.drop(centers[centers.isna()].index)
centers = centers.astype(int)
- residues = to_fit.rolling(pd.Timedelta(winsz), closed="both", min_periods=min_periods).apply(
- polyRollerIrregular, args=(centers, polydeg)
- )
+ residues = to_fit.rolling(
+ pd.Timedelta(winsz), closed="both", min_periods=min_periods
+ ).apply(polyRollerIrregular, args=(centers, polydeg))
def center_func(x, y=centers):
pos = x.index[int(len(x) - y[x.index[-1]])]
return y.index.get_loc(pos)
- centers_iloc = centers.rolling(winsz, closed="both").apply(center_func, raw=False).astype(int)
+ centers_iloc = (
+ centers.rolling(winsz, closed="both")
+ .apply(center_func, raw=False)
+ .astype(int)
+ )
temp = residues.copy()
for k in centers_iloc.iteritems():
residues.iloc[k[1]] = temp[k[0]]
- residues[residues.index[0]: residues.index[centers_iloc[0]]] = np.nan
- residues[residues.index[centers_iloc[-1]]: residues.index[-1]] = np.nan
+ residues[residues.index[0] : residues.index[centers_iloc[0]]] = np.nan
+ residues[residues.index[centers_iloc[-1]] : residues.index[-1]] = np.nan
else:
if isinstance(winsz, str):
winsz = pd.Timedelta(winsz) // regular
@@ -153,9 +164,9 @@ def fitPolynomial(
center_index = winsz // 2
if min_periods < winsz:
if min_periods > 0:
- to_fit = to_fit.rolling(winsz, min_periods=min_periods, center=True).apply(
- lambda x, y: x[y], raw=True, args=(center_index,)
- )
+ to_fit = to_fit.rolling(
+ winsz, min_periods=min_periods, center=True
+ ).apply(lambda x, y: x[y], raw=True, args=(center_index,))
# we need a missing value marker that is not nan,
# because nan values dont get passed by pandas rolling method
@@ -175,7 +186,9 @@ def fitPolynomial(
residues = residues.shift(-int(center_index))
else:
residues = to_fit.rolling(winsz, center=True).apply(
- polyRoller, args=(miss_marker, val_range, center_index, polydeg), raw=True
+ polyRoller,
+ args=(miss_marker, val_range, center_index, polydeg),
+ raw=True,
)
residues[na_mask] = np.nan
else:
@@ -192,7 +205,9 @@ def fitPolynomial(
residues = residues.shift(-int(center_index))
else:
residues = to_fit.rolling(winsz, center=True).apply(
- polyRollerNoMissing, args=(val_range, center_index, polydeg), raw=True
+ polyRollerNoMissing,
+ args=(val_range, center_index, polydeg),
+ raw=True,
)
if return_residues:
diff --git a/saqc/funcs/drift.py b/saqc/funcs/drift.py
index 86b9534ce..07a6527cd 100644
--- a/saqc/funcs/drift.py
+++ b/saqc/funcs/drift.py
@@ -24,22 +24,27 @@ from saqc.lib.tools import detectDeviants
from saqc.lib.types import FreqString, ColumnName, CurveFitter, TimestampColumnName
from saqc.lib.ts_operators import expModelFunc, expDriftModel, linearDriftModel
-LinkageString = Literal["single", "complete", "average", "weighted", "centroid", "median", "ward"]
+LinkageString = Literal[
+ "single", "complete", "average", "weighted", "centroid", "median", "ward"
+]
-@register(masking='all', module="drift")
+@register(masking="all", module="drift")
def flagDriftFromNorm(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- fields: Sequence[ColumnName],
- segment_freq: FreqString,
- norm_spread: float,
- norm_frac: float = 0.5,
- metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
- linkage_method: LinkageString = "single",
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ fields: Sequence[ColumnName],
+ segment_freq: FreqString,
+ norm_spread: float,
+ norm_frac: float = 0.5,
+ metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(
+ np.array([x, y]), metric="cityblock"
+ )
+ / len(x),
+ linkage_method: LinkageString = "single",
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
The function flags value courses that significantly deviate from a group of normal value courses.
@@ -136,7 +141,9 @@ def flagDriftFromNorm(
if segment[1].shape[0] <= 1:
continue
- drifters = detectDeviants(segment[1], metric, norm_spread, norm_frac, linkage_method, 'variables')
+ drifters = detectDeviants(
+ segment[1], metric, norm_spread, norm_frac, linkage_method, "variables"
+ )
for var in drifters:
flags[segment[1].index, fields[var]] = flag
@@ -144,17 +151,20 @@ def flagDriftFromNorm(
return data, flags
-@register(masking='all', module="drift")
+@register(masking="all", module="drift")
def flagDriftFromReference(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- fields: Sequence[ColumnName],
- segment_freq: FreqString,
- thresh: float,
- metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ fields: Sequence[ColumnName],
+ segment_freq: FreqString,
+ thresh: float,
+ metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(
+ np.array([x, y]), metric="cityblock"
+ )
+ / len(x),
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
The function flags value courses that deviate from a reference course by a margin exceeding a certain threshold.
@@ -222,20 +232,23 @@ def flagDriftFromReference(
return data, flags
-@register(masking='all', module="drift")
+@register(masking="all", module="drift")
def flagDriftFromScaledNorm(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- fields_scale1: Sequence[ColumnName],
- fields_scale2: Sequence[ColumnName],
- segment_freq: FreqString,
- norm_spread: float,
- norm_frac: float = 0.5,
- metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(np.array([x, y]), metric='cityblock') / len(x),
- linkage_method: LinkageString = "single",
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ fields_scale1: Sequence[ColumnName],
+ fields_scale2: Sequence[ColumnName],
+ segment_freq: FreqString,
+ norm_spread: float,
+ norm_frac: float = 0.5,
+ metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: pdist(
+ np.array([x, y]), metric="cityblock"
+ )
+ / len(x),
+ linkage_method: LinkageString = "single",
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
The function linearly rescales one set of variables to another set of variables with a different scale and then
@@ -311,7 +324,9 @@ def flagDriftFromScaledNorm(
for field1 in fields_scale1:
for field2 in fields_scale2:
- slope, intercept, *_ = stats.linregress(data_to_flag[field1], data_to_flag[field2])
+ slope, intercept, *_ = stats.linregress(
+ data_to_flag[field1], data_to_flag[field2]
+ )
convert_slope.append(slope)
convert_intercept.append(intercept)
@@ -332,7 +347,9 @@ def flagDriftFromScaledNorm(
if segment[1].shape[0] <= 1:
continue
- drifters = detectDeviants(segment[1], metric, norm_spread, norm_frac, linkage_method, 'variables')
+ drifters = detectDeviants(
+ segment[1], metric, norm_spread, norm_frac, linkage_method, "variables"
+ )
for var in drifters:
flags[segment[1].index, fields[var]] = flag
@@ -340,17 +357,17 @@ def flagDriftFromScaledNorm(
return data, flags
-@register(masking='all', module="drift")
+@register(masking="all", module="drift")
def correctDrift(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- maint_data_field: ColumnName,
- driftModel: Callable[..., float],
- cal_mean: int = 5,
- flag_maint_period: bool = False,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ maint_data_field: ColumnName,
+ driftModel: Callable[..., float],
+ cal_mean: int = 5,
+ flag_maint_period: bool = False,
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
The function corrects drifting behavior.
@@ -449,7 +466,9 @@ def correctDrift(
for k in range(0, maint_data.shape[0] - 1):
# assign group numbers for the timespans in between one maintenance ending and the beginning of the next
# maintenance time itself remains np.nan assigned
- drift_frame.loc[maint_data.values[k]: pd.Timestamp(maint_data.index[k + 1]), "drift_group"] = k
+ drift_frame.loc[
+ maint_data.values[k] : pd.Timestamp(maint_data.index[k + 1]), "drift_group"
+ ] = k
# define target values for correction
drift_grouper = drift_frame.groupby("drift_group")
@@ -457,7 +476,9 @@ def correctDrift(
for k, group in drift_grouper:
data_series = group[to_correct.name]
- data_fit, data_shiftTarget = _driftFit(data_series, shift_targets.loc[k, :][0], cal_mean, driftModel)
+ data_fit, data_shiftTarget = _driftFit(
+ data_series, shift_targets.loc[k, :][0], cal_mean, driftModel
+ )
data_fit = pd.Series(data_fit, index=group.index)
data_shiftTarget = pd.Series(data_shiftTarget, index=group.index)
data_shiftVektor = data_shiftTarget - data_fit
@@ -475,16 +496,16 @@ def correctDrift(
return data, flags
-@register(masking='all', module="drift")
+@register(masking="all", module="drift")
def correctRegimeAnomaly(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- cluster_field: ColumnName,
- model: CurveFitter,
- regime_transmission: Optional[FreqString] = None,
- x_date: bool = False,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ cluster_field: ColumnName,
+ model: CurveFitter,
+ regime_transmission: Optional[FreqString] = None,
+ x_date: bool = False,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
Function fits the passed model to the different regimes in data[field] and tries to correct
@@ -549,8 +570,8 @@ def correctRegimeAnomaly(
ydata = regime.values
valid_mask = ~np.isnan(ydata)
if regime_transmission is not None:
- valid_mask &= (xdata > xdata[0] + regime_transmission)
- valid_mask &= (xdata < xdata[-1] - regime_transmission)
+ valid_mask &= xdata > xdata[0] + regime_transmission
+ valid_mask &= xdata < xdata[-1] - regime_transmission
try:
p, *_ = curve_fit(model, xdata[valid_mask], ydata[valid_mask])
except (RuntimeError, ValueError):
@@ -561,12 +582,18 @@ def correctRegimeAnomaly(
first_normal = unique_successive > 0
first_valid = np.array(
- [~pd.isna(para_dict[unique_successive[i]]).any() for i in range(0, unique_successive.shape[0])])
+ [
+ ~pd.isna(para_dict[unique_successive[i]]).any()
+ for i in range(0, unique_successive.shape[0])
+ ]
+ )
first_valid = np.where(first_normal & first_valid)[0][0]
last_valid = 1
for k in range(0, unique_successive.shape[0]):
- if unique_successive[k] < 0 & (not pd.isna(para_dict[unique_successive[k]]).any()):
+ if unique_successive[k] < 0 & (
+ not pd.isna(para_dict[unique_successive[k]]).any()
+ ):
ydata = data_ser[regimes.groups[unique_successive[k]]].values
xdata = x_dict[unique_successive[k]]
ypara = para_dict[unique_successive[k]]
@@ -588,17 +615,17 @@ def correctRegimeAnomaly(
return data, flags
-@register(masking='all', module="drift")
+@register(masking="all", module="drift")
def correctOffset(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- max_mean_jump: float,
- normal_spread: float,
- search_winsz: FreqString,
- min_periods: int,
- regime_transmission: Optional[FreqString] = None,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ max_mean_jump: float,
+ normal_spread: float,
+ search_winsz: FreqString,
+ min_periods: int,
+ regime_transmission: Optional[FreqString] = None,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
Parameters
@@ -633,21 +660,28 @@ def correctOffset(
flags : saqc.Flags
The quality flags of data
"""
- data, flags = copy(data, field, flags, field + '_CPcluster')
+ data, flags = copy(data, field, flags, field + "_CPcluster")
data, flags = assignChangePointCluster(
- data, field + '_CPcluster', flags,
+ data,
+ field + "_CPcluster",
+ flags,
lambda x, y: np.abs(np.mean(x) - np.mean(y)),
lambda x, y: max_mean_jump,
bwd_window=search_winsz,
- min_periods_bwd=min_periods
+ min_periods_bwd=min_periods,
+ )
+ data, flags = assignRegimeAnomaly(
+ data, field, flags, field + "_CPcluster", normal_spread
)
- data, flags = assignRegimeAnomaly(data, field, flags, field + '_CPcluster', normal_spread)
data, flags = correctRegimeAnomaly(
- data, field, flags, field + '_CPcluster',
+ data,
+ field,
+ flags,
+ field + "_CPcluster",
lambda x, p1: np.array([p1] * x.shape[0]),
- regime_transmission=regime_transmission
+ regime_transmission=regime_transmission,
)
- data, flags = drop(data, field + '_CPcluster', flags)
+ data, flags = drop(data, field + "_CPcluster", flags)
return data, flags
@@ -663,14 +697,16 @@ def _driftFit(x, shift_target, cal_mean, driftModel):
dataFitFunc = functools.partial(driftModel, origin=origin_mean, target=target_mean)
# if drift model has free parameters:
try:
- # try fitting free parameters
- fit_paras, *_ = curve_fit(dataFitFunc, x_data, y_data)
- data_fit = dataFitFunc(x_data, *fit_paras)
- data_shift = driftModel(x_data, *fit_paras, origin=origin_mean, target=shift_target)
+ # try fitting free parameters
+ fit_paras, *_ = curve_fit(dataFitFunc, x_data, y_data)
+ data_fit = dataFitFunc(x_data, *fit_paras)
+ data_shift = driftModel(
+ x_data, *fit_paras, origin=origin_mean, target=shift_target
+ )
except RuntimeError:
- # if fit fails -> make no correction
- data_fit = np.array([0] * len(x_data))
- data_shift = np.array([0] * len(x_data))
+ # if fit fails -> make no correction
+ data_fit = np.array([0] * len(x_data))
+ data_shift = np.array([0] * len(x_data))
# when there are no free parameters in the model:
except ValueError:
data_fit = dataFitFunc(x_data)
@@ -679,18 +715,20 @@ def _driftFit(x, shift_target, cal_mean, driftModel):
return data_fit, data_shift
-@register(masking='all', module="drift")
+@register(masking="all", module="drift")
def flagRegimeAnomaly(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- cluster_field: ColumnName,
- norm_spread: float,
- linkage_method: LinkageString = "single",
- metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: np.abs(np.nanmean(x) - np.nanmean(y)),
- norm_frac: float = 0.5,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ cluster_field: ColumnName,
+ norm_spread: float,
+ linkage_method: LinkageString = "single",
+ metric: Callable[[np.ndarray, np.ndarray], float] = lambda x, y: np.abs(
+ np.nanmean(x) - np.nanmean(y)
+ ),
+ norm_frac: float = 0.5,
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
A function to flag values belonging to an anomalous regime regarding modelling regimes of field.
@@ -739,7 +777,9 @@ def flagRegimeAnomaly(
Flags values may have changed, relatively to the flags input.
"""
return assignRegimeAnomaly(
- data, field, flags,
+ data,
+ field,
+ flags,
cluster_field,
norm_spread,
linkage_method=linkage_method,
@@ -752,20 +792,22 @@ def flagRegimeAnomaly(
)
-@register(masking='all', module="drift")
+@register(masking="all", module="drift")
def assignRegimeAnomaly(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- cluster_field: ColumnName,
- norm_spread: float,
- linkage_method: LinkageString = "single",
- metric: Callable[[np.array, np.array], float] = lambda x, y: np.abs(np.nanmean(x) - np.nanmean(y)),
- norm_frac: float = 0.5,
- set_cluster: bool = True,
- set_flags: bool = False,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ cluster_field: ColumnName,
+ norm_spread: float,
+ linkage_method: LinkageString = "single",
+ metric: Callable[[np.array, np.array], float] = lambda x, y: np.abs(
+ np.nanmean(x) - np.nanmean(y)
+ ),
+ norm_frac: float = 0.5,
+ set_cluster: bool = True,
+ set_flags: bool = False,
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
A function to detect values belonging to an anomalous regime regarding modelling regimes of field.
@@ -824,7 +866,9 @@ def assignRegimeAnomaly(
series = data[cluster_field]
cluster = np.unique(series)
cluster_dios = DictOfSeries({i: data[field][series == i] for i in cluster})
- plateaus = detectDeviants(cluster_dios, metric, norm_spread, norm_frac, linkage_method, 'samples')
+ plateaus = detectDeviants(
+ cluster_dios, metric, norm_spread, norm_frac, linkage_method, "samples"
+ )
if set_flags:
for p in plateaus:
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index d08bb3142..29ccf84a0 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -11,9 +11,9 @@ from saqc.core import register, Flags
import warnings
-@register(masking='field', module="flagtools")
+@register(masking="field", module="flagtools")
def forceFlags(
- data: DictOfSeries, field: ColumnName, flags: Flags, flag: float = BAD, **kwargs
+ data: DictOfSeries, field: ColumnName, flags: Flags, flag: float = BAD, **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
Set whole column to a flag value.
@@ -46,8 +46,10 @@ def forceFlags(
# masking='none' is sufficient because call is redirected
-@register(masking='none', module="flagtools")
-def clearFlags(data: DictOfSeries, field: ColumnName, flags: Flags, **kwargs) -> Tuple[DictOfSeries, Flags]:
+@register(masking="none", module="flagtools")
+def clearFlags(
+ data: DictOfSeries, field: ColumnName, flags: Flags, **kwargs
+) -> Tuple[DictOfSeries, Flags]:
"""
Set whole column to UNFLAGGED.
@@ -72,17 +74,17 @@ def clearFlags(data: DictOfSeries, field: ColumnName, flags: Flags, **kwargs) ->
forceFlags : set whole column to a flag value
flagUnflagged : set flag value at all unflagged positions
"""
- if 'flag' in kwargs:
+ if "flag" in kwargs:
kwargs = {**kwargs} # copy
- flag = kwargs.pop('flag')
- warnings.warn(f'`flag={flag}` is ignored here.')
+ flag = kwargs.pop("flag")
+ warnings.warn(f"`flag={flag}` is ignored here.")
return forceFlags(data, field, flags, flag=UNFLAGGED, **kwargs)
-@register(masking='field', module="flagtools")
+@register(masking="field", module="flagtools")
def flagUnflagged(
- data: DictOfSeries, field: ColumnName, flags: Flags, flag: float = BAD, **kwargs
+ data: DictOfSeries, field: ColumnName, flags: Flags, flag: float = BAD, **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
Function sets a flag at all unflagged positions.
@@ -117,14 +119,16 @@ def flagUnflagged(
return data, flags
-@register(masking='field', module="flagtools")
+@register(masking="field", module="flagtools")
def flagManual(
- data: DictOfSeries, field: ColumnName, flags: Flags,
- mdata: Union[pd.Series, pd.DataFrame, DictOfSeries],
- mflag: Any = 1,
- method: Literal["plain", "ontime", "left-open", "right-open"] = 'plain',
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ mdata: Union[pd.Series, pd.DataFrame, DictOfSeries],
+ mflag: Any = 1,
+ method: Literal["plain", "ontime", "left-open", "right-open"] = "plain",
+ flag: float = BAD,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
Flag data by given, "manually generated" data.
@@ -259,8 +263,10 @@ def flagManual(
return data, flags
-@register(masking='none', module="flagtools")
-def flagDummy(data: DictOfSeries, field: ColumnName, flags: Flags, **kwargs) -> Tuple[DictOfSeries, Flags]:
+@register(masking="none", module="flagtools")
+def flagDummy(
+ data: DictOfSeries, field: ColumnName, flags: Flags, **kwargs
+) -> Tuple[DictOfSeries, Flags]:
"""
Function does nothing but returning data and flags.
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index 93a7eec9b..86cb2b572 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -16,11 +16,11 @@ from saqc.core.visitor import ENVIRONMENT
import operator as op
-_OP = {'<': op.lt, '<=': op.le, '==': op.eq, '!=': op.ne, '>': op.gt, '>=': op.ge}
+_OP = {"<": op.lt, "<=": op.le, "==": op.eq, "!=": op.ne, ">": op.gt, ">=": op.ge}
def _dslIsFlagged(
- flags: Flags, var: pd.Series, flag: float = None, comparator: str = None
+ flags: Flags, var: pd.Series, flag: float = None, comparator: str = None
) -> Union[pd.Series, DictOfSeries]:
"""
helper function for `flag`
@@ -37,20 +37,25 @@ def _dslIsFlagged(
"""
if flag is None:
if comparator is not None:
- raise ValueError('if `comparator` is used, explicitly pass a `flag` level.')
+ raise ValueError("if `comparator` is used, explicitly pass a `flag` level.")
flag = UNFLAGGED
- comparator = '>'
+ comparator = ">"
# default
if comparator is None:
- comparator = '>='
+ comparator = ">="
_op = _OP[comparator]
return _op(flags[var.name], flag)
-def _execGeneric(flags: Flags, data: DictOfSeries, func: Callable[[pd.Series], pd.Series], field: str,
- nodata: float) -> pd.Series:
+def _execGeneric(
+ flags: Flags,
+ data: DictOfSeries,
+ func: Callable[[pd.Series], pd.Series],
+ field: str,
+ nodata: float,
+) -> pd.Series:
# TODO:
# - check series.index compatibility
# - field is only needed to translate 'this' parameters
@@ -79,14 +84,14 @@ def _execGeneric(flags: Flags, data: DictOfSeries, func: Callable[[pd.Series], p
return func(*args)
-@register(masking='all', module="generic")
+@register(masking="all", module="generic")
def process(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- func: Callable[[pd.Series], pd.Series],
- nodata: float = np.nan,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ func: Callable[[pd.Series], pd.Series],
+ nodata: float = np.nan,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
generate/process data with generically defined functions.
@@ -148,15 +153,15 @@ def process(
return data, flags
-@register(masking='all', module="generic")
+@register(masking="all", module="generic")
def flag(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- func: Callable[[pd.Series], pd.Series],
- nodata: float = np.nan,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ func: Callable[[pd.Series], pd.Series],
+ nodata: float = np.nan,
+ flag: float = BAD,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
# TODO : fix docstring, check if all still works
"""
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index 1c5131257..9672d6c55 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -13,20 +13,35 @@ from saqc.core.history import applyFunctionOnHistory
from saqc.lib.ts_operators import interpolateNANs
_SUPPORTED_METHODS = Literal[
- "linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric",
- "polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"
+ "linear",
+ "time",
+ "nearest",
+ "zero",
+ "slinear",
+ "quadratic",
+ "cubic",
+ "spline",
+ "barycentric",
+ "polynomial",
+ "krogh",
+ "piecewise_polynomial",
+ "spline",
+ "pchip",
+ "akima",
]
-@register(masking='field', module="interpolation")
+@register(masking="field", module="interpolation")
def interpolateByRolling(
- data: DictOfSeries, field: str, flags: Flags,
- winsz: Union[str, int],
- func: Callable[[pd.Series], float] = np.median,
- center: bool = True,
- min_periods: int = 0,
- flag: float = UNFLAGGED,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ winsz: Union[str, int],
+ func: Callable[[pd.Series], float] = np.median,
+ center: bool = True,
+ min_periods: int = 0,
+ flag: float = UNFLAGGED,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
Interpolates nan-values in the data by assigning them the aggregation result of the window surrounding them.
@@ -91,17 +106,17 @@ def interpolateByRolling(
return data, flags
-@register(masking='field', module="interpolation")
+@register(masking="field", module="interpolation")
def interpolateInvalid(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- method: _SUPPORTED_METHODS,
- inter_order: int = 2,
- inter_limit: int = 2,
- downgrade_interpolation: bool = False,
- flag: float = UNFLAGGED,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ method: _SUPPORTED_METHODS,
+ inter_order: int = 2,
+ inter_limit: int = 2,
+ downgrade_interpolation: bool = False,
+ flag: float = UNFLAGGED,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
Function to interpolate nan values in the data.
@@ -153,7 +168,7 @@ def interpolateInvalid(
method,
order=inter_order,
inter_limit=inter_limit,
- downgrade_interpolation=downgrade_interpolation
+ downgrade_interpolation=downgrade_interpolation,
)
interpolated = data[field].isna() & inter_data.notna()
@@ -175,17 +190,17 @@ def _resampleOverlapping(data: pd.Series, freq: str, fill_value):
return data.fillna(fill_value).astype(dtype)
-@register(masking='none', module="interpolation")
+@register(masking="none", module="interpolation")
def interpolateIndex(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- freq: str,
- method: _SUPPORTED_METHODS,
- inter_order: int = 2,
- inter_limit: int = 2,
- downgrade_interpolation: bool = False,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ freq: str,
+ method: _SUPPORTED_METHODS,
+ inter_order: int = 2,
+ inter_limit: int = 2,
+ downgrade_interpolation: bool = False,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
Function to interpolate the data at regular (equidistant) timestamps (or Grid points).
@@ -243,7 +258,7 @@ def interpolateIndex(
start, end = datcol.index[0].floor(freq), datcol.index[-1].ceil(freq)
grid_index = pd.date_range(start=start, end=end, freq=freq, name=datcol.index.name)
- flagged = _isflagged(flags[field], kwargs['to_mask'])
+ flagged = _isflagged(flags[field], kwargs["to_mask"])
# drop all points that hold no relevant grid information
datcol = datcol[~flagged].dropna()
@@ -275,9 +290,11 @@ def interpolateIndex(
# do the reshaping on the history
flags.history[field] = applyFunctionOnHistory(
flags.history[field],
- hist_func=_resampleOverlapping, hist_kws=dict(freq=freq, fill_value=UNFLAGGED),
- mask_func=_resampleOverlapping, mask_kws=dict(freq=freq, fill_value=False),
- last_column='dummy'
+ hist_func=_resampleOverlapping,
+ hist_kws=dict(freq=freq, fill_value=UNFLAGGED),
+ mask_func=_resampleOverlapping,
+ mask_kws=dict(freq=freq, fill_value=False),
+ last_column="dummy",
)
return data, flags
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index b16e93d99..e3d473ef7 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -19,17 +19,17 @@ from saqc.funcs.scores import assignKNNScore
import saqc.lib.ts_operators as ts_ops
-@register(masking='field', module="outliers")
+@register(masking="field", module="outliers")
def flagByStray(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- partition_freq: Optional[Union[IntegerWindow, FreqString]] = None,
- partition_min: int = 11,
- iter_start: float = 0.5,
- alpha: float = 0.05,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ partition_freq: Optional[Union[IntegerWindow, FreqString]] = None,
+ partition_min: int = 11,
+ iter_start: float = 0.5,
+ alpha: float = 0.05,
+ flag: float = BAD,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
Flag outliers in 1-dimensional (score) data with the STRAY Algorithm.
@@ -88,8 +88,12 @@ def flagByStray(
partitions = scores.groupby(pd.Grouper(freq=partition_freq))
else:
- grouper_series = pd.Series(data=np.arange(0, scores.shape[0]), index=scores.index)
- grouper_series = grouper_series.transform(lambda x: int(np.floor(x / partition_freq)))
+ grouper_series = pd.Series(
+ data=np.arange(0, scores.shape[0]), index=scores.index
+ )
+ grouper_series = grouper_series.transform(
+ lambda x: int(np.floor(x / partition_freq))
+ )
partitions = scores.groupby(grouper_series)
# calculate flags for every partition
@@ -124,17 +128,17 @@ def flagByStray(
def _evalStrayLabels(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- fields: Sequence[str],
- reduction_range: Optional[str] = None,
- reduction_drop_flagged: bool = False, # TODO: still a case ?
- reduction_thresh: float = 3.5,
- reduction_min_periods: int = 1,
- at_least_one: bool = True,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ fields: Sequence[str],
+ reduction_range: Optional[str] = None,
+ reduction_drop_flagged: bool = False, # TODO: still a case ?
+ reduction_thresh: float = 3.5,
+ reduction_min_periods: int = 1,
+ at_least_one: bool = True,
+ flag: float = BAD,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
The function "reduces" an observations flag to components of it, by applying MAD (See references)
@@ -190,7 +194,10 @@ def _evalStrayLabels(
for var in fields:
for index in enumerate(to_flag_frame.index):
- index_slice = slice(index[1] - pd.Timedelta(reduction_range), index[1] + pd.Timedelta(reduction_range))
+ index_slice = slice(
+ index[1] - pd.Timedelta(reduction_range),
+ index[1] + pd.Timedelta(reduction_range),
+ )
test_slice = val_frame[var][index_slice].dropna()
# check, wheather value under test is sufficiently centered:
@@ -198,25 +205,30 @@ def _evalStrayLabels(
last = test_slice.last_valid_index()
min_range = pd.Timedelta(reduction_range) / 4
- if pd.Timedelta(index[1] - first) < min_range or pd.Timedelta(last - index[1]) < min_range:
+ if (
+ pd.Timedelta(index[1] - first) < min_range
+ or pd.Timedelta(last - index[1]) < min_range
+ ):
polydeg = 0
else:
polydeg = 2
if reduction_drop_flagged:
- test_slice = test_slice.drop(to_flag_frame.index, errors='ignore')
+ test_slice = test_slice.drop(to_flag_frame.index, errors="ignore")
if test_slice.shape[0] < reduction_min_periods:
to_flag_frame.loc[index[1], var] = True
continue
- x = (test_slice.index.values.astype(float))
+ x = test_slice.index.values.astype(float)
x_0 = x[0]
x = (x - x_0) / 10 ** 12
polyfitted = poly.polyfit(y=test_slice.values, x=x, deg=polydeg)
- testval = poly.polyval((float(index[1].to_numpy()) - x_0) / 10 ** 12, polyfitted)
+ testval = poly.polyval(
+ (float(index[1].to_numpy()) - x_0) / 10 ** 12, polyfitted
+ )
testval = val_frame[var][index[1]] - testval
resids = test_slice.values - poly.polyval(x, polyfitted)
@@ -237,7 +249,14 @@ def _evalStrayLabels(
return data, flags
-def _expFit(val_frame, scoring_method="kNNMaxGap", n_neighbors=10, iter_start=0.5, alpha=0.05, bin_frac=10):
+def _expFit(
+ val_frame,
+ scoring_method="kNNMaxGap",
+ n_neighbors=10,
+ iter_start=0.5,
+ alpha=0.05,
+ bin_frac=10,
+):
"""
Find outliers in multi dimensional observations.
@@ -292,8 +311,19 @@ def _expFit(val_frame, scoring_method="kNNMaxGap", n_neighbors=10, iter_start=0.
# initialise sampling bins
if isinstance(bin_frac, int):
- binz = np.linspace(resids[0], resids[-1], 10 * int(np.ceil(data_len / bin_frac)))
- elif bin_frac in ["auto", "fd", "doane", "scott", "stone", "rice", "sturges", "sqrt"]:
+ binz = np.linspace(
+ resids[0], resids[-1], 10 * int(np.ceil(data_len / bin_frac))
+ )
+ elif bin_frac in [
+ "auto",
+ "fd",
+ "doane",
+ "scott",
+ "stone",
+ "rice",
+ "sturges",
+ "sqrt",
+ ]:
binz = np.histogram_bin_edges(resids, bins=bin_frac)
else:
raise ValueError(f"Can't interpret {bin_frac} as an binning technique.")
@@ -315,7 +345,8 @@ def _expFit(val_frame, scoring_method="kNNMaxGap", n_neighbors=10, iter_start=0.
upper_tail_index = int(np.floor(0.5 * hist_argmax + 0.5 * iter_max_bin_index))
resids_tail_index = findIndex(resids, binz[upper_tail_index], 0)
upper_tail_hist, bins = np.histogram(
- resids[resids_tail_index:iter_index], bins=binz[upper_tail_index: iter_max_bin_index + 1]
+ resids[resids_tail_index:iter_index],
+ bins=binz[upper_tail_index : iter_max_bin_index + 1],
)
while (test_val < crit_val) & (iter_index < resids.size - 1):
@@ -326,11 +357,15 @@ def _expFit(val_frame, scoring_method="kNNMaxGap", n_neighbors=10, iter_start=0.
if new_iter_max_bin_index == iter_max_bin_index:
upper_tail_hist[-1] += 1
else:
- upper_tail_hist = np.append(upper_tail_hist, np.zeros([new_iter_max_bin_index - iter_max_bin_index]))
+ upper_tail_hist = np.append(
+ upper_tail_hist, np.zeros([new_iter_max_bin_index - iter_max_bin_index])
+ )
upper_tail_hist[-1] += 1
iter_max_bin_index = new_iter_max_bin_index
- upper_tail_index_new = int(np.floor(0.5 * hist_argmax + 0.5 * iter_max_bin_index))
- upper_tail_hist = upper_tail_hist[upper_tail_index_new - upper_tail_index:]
+ upper_tail_index_new = int(
+ np.floor(0.5 * hist_argmax + 0.5 * iter_max_bin_index)
+ )
+ upper_tail_hist = upper_tail_hist[upper_tail_index_new - upper_tail_index :]
upper_tail_index = upper_tail_index_new
# fitting
@@ -348,26 +383,26 @@ def _expFit(val_frame, scoring_method="kNNMaxGap", n_neighbors=10, iter_start=0.
return val_frame.index[sorted_i[iter_index:]]
-@register(masking='all', module="outliers")
+@register(masking="all", module="outliers")
def flagMVScores(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- fields: Sequence[ColumnName],
- trafo: Callable[[pd.Series], pd.Series] = lambda x: x,
- alpha: float = 0.05,
- n_neighbors: int = 10,
- scoring_func: Callable[[pd.Series], float] = np.sum,
- iter_start: float = 0.5,
- stray_partition: Optional[Union[IntegerWindow, FreqString]] = None,
- stray_partition_min: int = 11,
- trafo_on_partition: bool = True,
- reduction_range: Optional[FreqString] = None,
- reduction_drop_flagged: bool = False, # TODO: still a case ?
- reduction_thresh: float = 3.5,
- reduction_min_periods: int = 1,
- flag: float = BAD,
- **kwargs,
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ fields: Sequence[ColumnName],
+ trafo: Callable[[pd.Series], pd.Series] = lambda x: x,
+ alpha: float = 0.05,
+ n_neighbors: int = 10,
+ scoring_func: Callable[[pd.Series], float] = np.sum,
+ iter_start: float = 0.5,
+ stray_partition: Optional[Union[IntegerWindow, FreqString]] = None,
+ stray_partition_min: int = 11,
+ trafo_on_partition: bool = True,
+ reduction_range: Optional[FreqString] = None,
+ reduction_drop_flagged: bool = False, # TODO: still a case ?
+ reduction_thresh: float = 3.5,
+ reduction_min_periods: int = 1,
+ flag: float = BAD,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
The algorithm implements a 3-step outlier detection procedure for simultaneously flagging of higher dimensional
@@ -474,54 +509,64 @@ def flagMVScores(
overview over the `stray` algorithm.
"""
data, flags = assignKNNScore(
- data, 'dummy', flags,
+ data,
+ "dummy",
+ flags,
fields=fields,
n_neighbors=n_neighbors,
trafo=trafo,
trafo_on_partition=trafo_on_partition,
scoring_func=scoring_func,
- target_field='kNN_scores',
+ target_field="kNN_scores",
partition_freq=stray_partition,
- kNN_algorithm='ball_tree',
- partition_min=stray_partition_min, **kwargs)
+ kNN_algorithm="ball_tree",
+ partition_min=stray_partition_min,
+ **kwargs,
+ )
data, flags = flagByStray(
- data, 'kNN_scores', flags,
+ data,
+ "kNN_scores",
+ flags,
partition_freq=stray_partition,
partition_min=stray_partition_min,
iter_start=iter_start,
alpha=alpha,
flag=flag,
- **kwargs)
+ **kwargs,
+ )
data, flags = _evalStrayLabels(
- data, 'kNN_scores', flags,
+ data,
+ "kNN_scores",
+ flags,
fields=fields,
reduction_range=reduction_range,
reduction_drop_flagged=reduction_drop_flagged,
reduction_thresh=reduction_thresh,
reduction_min_periods=reduction_min_periods,
flag=flag,
- **kwargs)
+ **kwargs,
+ )
return data, flags
-@register(masking='field', module="outliers")
+@register(masking="field", module="outliers")
def flagRaise(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- thresh: float,
- raise_window: FreqString,
- intended_freq: FreqString,
- average_window: Optional[FreqString] = None,
- mean_raise_factor: float = 2.,
- min_slope: Optional[float] = None,
- min_slope_weight: float = 0.8,
- numba_boost: bool = True, # TODO: rm, not a user decision
- flag: float = BAD,
- **kwargs,
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ thresh: float,
+ raise_window: FreqString,
+ intended_freq: FreqString,
+ average_window: Optional[FreqString] = None,
+ mean_raise_factor: float = 2.0,
+ min_slope: Optional[float] = None,
+ min_slope_weight: float = 0.8,
+ numba_boost: bool = True, # TODO: rm, not a user decision
+ flag: float = BAD,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
The function flags raises and drops in value courses, that exceed a certain threshold
@@ -624,7 +669,9 @@ def flagRaise(
if numba_boost:
raise_check = numba.jit(raise_check, nopython=True)
- raise_series = raise_series.apply(raise_check, args=(thresh,), raw=True, engine="numba")
+ raise_series = raise_series.apply(
+ raise_check, args=(thresh,), raw=True, engine="numba"
+ )
else:
raise_series = raise_series.apply(raise_check, args=(thresh,), raw=True)
@@ -634,39 +681,52 @@ def flagRaise(
# "unflag" values of insufficient deviation to their predecessors
if min_slope is not None:
w_mask = (
- pd.Series(dataseries.index).diff().dt.total_seconds() / intended_freq.total_seconds()
- ) > min_slope_weight
+ pd.Series(dataseries.index).diff().dt.total_seconds()
+ / intended_freq.total_seconds()
+ ) > min_slope_weight
slope_mask = np.abs(dataseries.diff()) < min_slope
to_unflag = raise_series.notna() & w_mask.values & slope_mask
raise_series[to_unflag] = np.nan
# calculate and apply the weighted mean weights (pseudo-harmonization):
weights = (
- pd.Series(dataseries.index).diff(periods=2).shift(-1).dt.total_seconds() / intended_freq.total_seconds() / 2
+ pd.Series(dataseries.index).diff(periods=2).shift(-1).dt.total_seconds()
+ / intended_freq.total_seconds()
+ / 2
)
- weights.iloc[0] = 0.5 + (dataseries.index[1] - dataseries.index[0]).total_seconds() / (
- intended_freq.total_seconds() * 2
- )
+ weights.iloc[0] = 0.5 + (
+ dataseries.index[1] - dataseries.index[0]
+ ).total_seconds() / (intended_freq.total_seconds() * 2)
- weights.iloc[-1] = 0.5 + (dataseries.index[-1] - dataseries.index[-2]).total_seconds() / (
- intended_freq.total_seconds() * 2
- )
+ weights.iloc[-1] = 0.5 + (
+ dataseries.index[-1] - dataseries.index[-2]
+ ).total_seconds() / (intended_freq.total_seconds() * 2)
weights[weights > 1.5] = 1.5
weights.index = dataseries.index
weighted_data = dataseries.mul(weights)
# rolling weighted mean calculation
- weighted_rolling_mean = weighted_data.rolling(average_window, min_periods=2, closed="both")
+ weighted_rolling_mean = weighted_data.rolling(
+ average_window, min_periods=2, closed="both"
+ )
weights_rolling_sum = weights.rolling(average_window, min_periods=2, closed="both")
if numba_boost:
custom_rolling_mean = numba.jit(custom_rolling_mean, nopython=True)
- weighted_rolling_mean = weighted_rolling_mean.apply(custom_rolling_mean, raw=True, engine="numba")
- weights_rolling_sum = weights_rolling_sum.apply(custom_rolling_mean, raw=True, engine="numba")
+ weighted_rolling_mean = weighted_rolling_mean.apply(
+ custom_rolling_mean, raw=True, engine="numba"
+ )
+ weights_rolling_sum = weights_rolling_sum.apply(
+ custom_rolling_mean, raw=True, engine="numba"
+ )
else:
- weighted_rolling_mean = weighted_rolling_mean.apply(custom_rolling_mean, raw=True)
- weights_rolling_sum = weights_rolling_sum.apply(custom_rolling_mean, raw=True, engine="numba")
+ weighted_rolling_mean = weighted_rolling_mean.apply(
+ custom_rolling_mean, raw=True
+ )
+ weights_rolling_sum = weights_rolling_sum.apply(
+ custom_rolling_mean, raw=True, engine="numba"
+ )
weighted_rolling_mean = weighted_rolling_mean / weights_rolling_sum
# check means against critical raise value:
@@ -677,15 +737,15 @@ def flagRaise(
return data, flags
-@register(masking='field', module="outliers")
+@register(masking="field", module="outliers")
def flagMAD(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- window: FreqString,
- z: float = 3.5,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ window: FreqString,
+ z: float = 3.5,
+ flag: float = BAD,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
The function represents an implementation of the modyfied Z-score outlier detection method.
@@ -743,18 +803,18 @@ def flagMAD(
return data, flags
-@register(masking='field', module="outliers")
+@register(masking="field", module="outliers")
def flagOffset(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- thresh: float,
- tolerance: float,
- window: Union[IntegerWindow, FreqString],
- rel_thresh: Optional[float] = None,
- numba_kickin: int = 200000, # TODO: rm, not a user decision
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ thresh: float,
+ tolerance: float,
+ window: Union[IntegerWindow, FreqString],
+ rel_thresh: Optional[float] = None,
+ numba_kickin: int = 200000, # TODO: rm, not a user decision
+ flag: float = BAD,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
A basic outlier test that is designed to work for harmonized and not harmonized data.
@@ -826,7 +886,9 @@ def flagOffset(
delta = getFreqDelta(dataseries.index)
window = delta * window
if not delta:
- raise TypeError('Only offset string defined window sizes allowed for irrgegularily sampled timeseries')
+ raise TypeError(
+ "Only offset string defined window sizes allowed for irrgegularily sampled timeseries"
+ )
# get all the entries preceding a significant jump
if thresh:
@@ -845,7 +907,9 @@ def flagOffset(
return data, flags
# get all the entries preceding a significant jump and its successors within "length" range
- to_roll = post_jumps.reindex(dataseries.index, method="bfill", tolerance=window, fill_value=False).dropna()
+ to_roll = post_jumps.reindex(
+ dataseries.index, method="bfill", tolerance=window, fill_value=False
+ ).dropna()
if rel_thresh:
@@ -856,7 +920,7 @@ def flagOffset(
initial = np.searchsorted(chunk_stair, 2)
if initial == len(chunk):
return 0
- if np.abs(chunk[- initial - 1] - chunk[-1]) < tol:
+ if np.abs(chunk[-initial - 1] - chunk[-1]) < tol:
return initial - 1
return 0
@@ -865,11 +929,13 @@ def flagOffset(
# define spike testing function to roll with (no rel_check):
def spikeTester(chunk, thresh=thresh, tol=tolerance):
# signum change!!!
- chunk_stair = (np.sign(chunk[-2] - chunk[-1]) * (chunk - chunk[-1]) < thresh)[::-1].cumsum()
+ chunk_stair = (
+ np.sign(chunk[-2] - chunk[-1]) * (chunk - chunk[-1]) < thresh
+ )[::-1].cumsum()
initial = np.searchsorted(chunk_stair, 2)
if initial == len(chunk):
return 0
- if np.abs(chunk[- initial - 1] - chunk[-1]) < tol:
+ if np.abs(chunk[-initial - 1] - chunk[-1]) < tol:
return initial - 1
return 0
@@ -877,8 +943,10 @@ def flagOffset(
roll_mask = pd.Series(False, index=to_roll.index)
roll_mask[post_jumps.index] = True
- roller = customRoller(to_roll, window=window, mask=roll_mask, min_periods=2, closed='both')
- engine = None if roll_mask.sum() < numba_kickin else 'numba'
+ roller = customRoller(
+ to_roll, window=window, mask=roll_mask, min_periods=2, closed="both"
+ )
+ engine = None if roll_mask.sum() < numba_kickin else "numba"
result = roller.apply(spikeTester, raw=True, engine=engine)
result.index = map_i[result.index]
@@ -892,7 +960,7 @@ def flagOffset(
k_r = int(result[k])
# validity check: plateuas start isnt another plateaus end:
if not flag_scopes[k - k_r - 1]:
- flag_scopes[(k - k_r):k] = True
+ flag_scopes[(k - k_r) : k] = True
return pd.Series(flag_scopes, index=result.index)
cresult = calcResult(result)
@@ -901,17 +969,17 @@ def flagOffset(
return data, flags
-@register(masking='field', module="outliers")
+@register(masking="field", module="outliers")
def flagByGrubbs(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- winsz: Union[FreqString, IntegerWindow],
- alpha: float = 0.05,
- min_periods: int = 8,
- check_lagged: bool = False,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ winsz: Union[FreqString, IntegerWindow],
+ alpha: float = 0.05,
+ min_periods: int = 8,
+ check_lagged: bool = False,
+ flag: float = BAD,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
The function flags values that are regarded outliers due to the grubbs test.
@@ -978,7 +1046,9 @@ def flagByGrubbs(
# period number defined test intervals
if isinstance(winsz, int):
- grouper_series = pd.Series(data=np.arange(0, datcol.shape[0]), index=datcol.index)
+ grouper_series = pd.Series(
+ data=np.arange(0, datcol.shape[0]), index=datcol.index
+ )
grouper_series_lagged = grouper_series + (winsz / 2)
grouper_series = grouper_series.transform(lambda x: x // winsz)
grouper_series_lagged = grouper_series_lagged.transform(lambda x: x // winsz)
@@ -991,7 +1061,9 @@ def flagByGrubbs(
for _, partition in partitions:
if partition.shape[0] > min_periods:
- detected = smirnov_grubbs.two_sided_test_indices(partition["data"].values, alpha=alpha)
+ detected = smirnov_grubbs.two_sided_test_indices(
+ partition["data"].values, alpha=alpha
+ )
detected = partition["ts"].iloc[detected]
to_flag[detected.index] = True
@@ -1000,7 +1072,9 @@ def flagByGrubbs(
for _, partition in partitions_lagged:
if partition.shape[0] > min_periods:
- detected = smirnov_grubbs.two_sided_test_indices(partition["data"].values, alpha=alpha)
+ detected = smirnov_grubbs.two_sided_test_indices(
+ partition["data"].values, alpha=alpha
+ )
detected = partition["ts"].iloc[detected]
to_flag_lagged[detected.index] = True
@@ -1010,15 +1084,15 @@ def flagByGrubbs(
return data, flags
-@register(masking='field', module="outliers")
+@register(masking="field", module="outliers")
def flagRange(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- min: float = -np.inf,
- max: float = np.inf,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ min: float = -np.inf,
+ max: float = np.inf,
+ flag: float = BAD,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
Function flags values not covered by the closed interval [`min`, `max`].
@@ -1053,16 +1127,16 @@ def flagRange(
return data, flags
-@register(masking='all', module="outliers")
+@register(masking="all", module="outliers")
def flagCrossStatistic(
- data: DictOfSeries,
- field: ColumnName,
- flags: Flags,
- fields: Sequence[ColumnName],
- thresh: float,
- cross_stat: Literal["modZscore", "Zscore"] = "modZscore",
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: ColumnName,
+ flags: Flags,
+ fields: Sequence[ColumnName],
+ thresh: float,
+ cross_stat: Literal["modZscore", "Zscore"] = "modZscore",
+ flag: float = BAD,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
Function checks for outliers relatively to the "horizontal" input data axis.
@@ -1113,16 +1187,24 @@ def flagCrossStatistic(
[1] https://www.itl.nist.gov/div898/handbook/eda/section3/eda35h.htm
"""
- df = data[fields].loc[data[fields].index_of('shared')].to_df()
+ df = data[fields].loc[data[fields].index_of("shared")].to_df()
if isinstance(cross_stat, str):
- if cross_stat == 'modZscore':
+ if cross_stat == "modZscore":
MAD_series = df.subtract(df.median(axis=1), axis=0).abs().median(axis=1)
- diff_scores = (0.6745 * (df.subtract(df.median(axis=1), axis=0))).divide(MAD_series, axis=0).abs()
-
- elif cross_stat == 'Zscore':
- diff_scores = df.subtract(df.mean(axis=1), axis=0).divide(df.std(axis=1), axis=0).abs()
+ diff_scores = (
+ (0.6745 * (df.subtract(df.median(axis=1), axis=0)))
+ .divide(MAD_series, axis=0)
+ .abs()
+ )
+
+ elif cross_stat == "Zscore":
+ diff_scores = (
+ df.subtract(df.mean(axis=1), axis=0)
+ .divide(df.std(axis=1), axis=0)
+ .abs()
+ )
else:
raise ValueError(cross_stat)
diff --git a/saqc/funcs/pattern.py b/saqc/funcs/pattern.py
index 564247488..ca59f1771 100644
--- a/saqc/funcs/pattern.py
+++ b/saqc/funcs/pattern.py
@@ -13,16 +13,16 @@ from saqc.core import register, Flags
from saqc.lib.tools import customRoller
-@register(masking='field', module="pattern")
+@register(masking="field", module="pattern")
def flagPatternByDTW(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- ref_field: str,
- widths: Sequence[int] = (1, 2, 4, 8),
- waveform: str = "mexh",
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ ref_field: str,
+ widths: Sequence[int] = (1, 2, 4, 8),
+ waveform: str = "mexh",
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
Pattern recognition via wavelets.
@@ -88,7 +88,9 @@ def flagPatternByDTW(
for i in range(len_width):
x = wavepower_ref[i]
y = wavepower_chunk[i]
- pval = permutation_test(x, y, method='approximate', num_rounds=200, func=func, seed=0)
+ pval = permutation_test(
+ x, y, method="approximate", num_rounds=200, func=func, seed=0
+ )
if min(pval, 1 - pval) > 0.01:
return True
return False
@@ -101,18 +103,18 @@ def flagPatternByDTW(
return data, flags
-@register(masking='field', module="pattern")
+@register(masking="field", module="pattern")
def flagPatternByWavelet(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- ref_field: str,
- max_distance: float = 0.03,
- normalize: bool = True,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ ref_field: str,
+ max_distance: float = 0.03,
+ normalize: bool = True,
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
- """ Pattern Recognition via Dynamic Time Warping.
+ """Pattern Recognition via Dynamic Time Warping.
The steps are:
1. work on chunks returned by a moving window
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index e157fd0c5..1623f5048 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -31,17 +31,17 @@ METHOD2ARGS = {
}
-@register(masking='none', module="resampling")
+@register(masking="none", module="resampling")
def aggregate(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- freq: str,
- value_func,
- flag_func: Callable[[pd.Series], float] = np.nanmax,
- method: Literal["fagg", "bagg", "nagg"] = "nagg",
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ freq: str,
+ value_func,
+ flag_func: Callable[[pd.Series], float] = np.nanmax,
+ method: Literal["fagg", "bagg", "nagg"] = "nagg",
+ flag: float = BAD,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
A method to "regularize" data by aggregating (resampling) data at a regular timestamp.
@@ -106,25 +106,23 @@ def aggregate(
Flags values and shape may have changed relatively to the flags input.
"""
- data, flags = copy(data, field, flags, field + '_original')
+ data, flags = copy(data, field, flags, field + "_original")
return resample(
- data, field, flags,
+ data,
+ field,
+ flags,
freq=freq,
agg_func=value_func,
flag_agg_func=flag_func,
method=method,
flag=flag,
- **kwargs
+ **kwargs,
)
-@register(masking='none', module="resampling")
+@register(masking="none", module="resampling")
def linear(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- freq: str,
- **kwargs
+ data: DictOfSeries, field: str, flags: Flags, freq: str, **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
A method to "regularize" data by interpolating linearly the data at regular timestamp.
@@ -165,19 +163,19 @@ def linear(
Flags values and shape may have changed relatively to the flags input.
"""
- data, flags = copy(data, field, flags, field + '_original')
+ data, flags = copy(data, field, flags, field + "_original")
return interpolateIndex(data, field, flags, freq, "time", **kwargs)
-@register(masking='none', module="resampling")
+@register(masking="none", module="resampling")
def interpolate(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- freq: str,
- method: _SUPPORTED_METHODS,
- order: int = 1,
- **kwargs,
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ freq: str,
+ method: _SUPPORTED_METHODS,
+ order: int = 1,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
A method to "regularize" data by interpolating the data at regular timestamp.
@@ -232,21 +230,27 @@ def interpolate(
Flags values and shape may have changed relatively to the flags input.
"""
- data, flags = copy(data, field, flags, field + '_original')
- return interpolateIndex(data, field, flags, freq, method=method, inter_order=order, **kwargs)
+ data, flags = copy(data, field, flags, field + "_original")
+ return interpolateIndex(
+ data, field, flags, freq, method=method, inter_order=order, **kwargs
+ )
-@register(masking='none', module="resampling")
+@register(masking="none", module="resampling")
def mapToOriginal(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- method: Literal[
- "inverse_fagg", "inverse_bagg", "inverse_nagg",
- "inverse_fshift", "inverse_bshift", "inverse_nshift",
- "inverse_interpolation"
- ],
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ method: Literal[
+ "inverse_fagg",
+ "inverse_bagg",
+ "inverse_nagg",
+ "inverse_fshift",
+ "inverse_bshift",
+ "inverse_nshift",
+ "inverse_interpolation",
+ ],
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
The Function function "undoes" regularization, by regaining the original data and projecting the
@@ -311,21 +315,23 @@ def mapToOriginal(
The quality flags of data
Flags values and shape may have changed relatively to the flags input.
"""
- newfield = str(field) + '_original'
- data, flags = reindexFlags(data, newfield, flags, method, source=field, to_mask=False)
+ newfield = str(field) + "_original"
+ data, flags = reindexFlags(
+ data, newfield, flags, method, source=field, to_mask=False
+ )
data, flags = drop(data, field, flags)
return rename(data, newfield, flags, field)
-@register(masking='none', module="resampling")
+@register(masking="none", module="resampling")
def shift(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- freq: str,
- method: Literal["fshift", "bshift", "nshift"] = "nshift",
- freq_check: Optional[Literal["check", "auto"]] = None, # TODO: not a user decision
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ freq: str,
+ method: Literal["fshift", "bshift", "nshift"] = "nshift",
+ freq_check: Optional[Literal["check", "auto"]] = None, # TODO: not a user decision
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
Function to shift data and flags to a regular (equidistant) timestamp grid, according to ``method``.
@@ -370,18 +376,20 @@ def shift(
The quality flags of data
Flags values and shape may have changed relatively to the flags input.
"""
- data, flags = copy(data, field, flags, field + '_original')
- return _shift(data, field, flags, freq, method=method, freq_check=freq_check, **kwargs)
+ data, flags = copy(data, field, flags, field + "_original")
+ return _shift(
+ data, field, flags, freq, method=method, freq_check=freq_check, **kwargs
+ )
def _shift(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- freq: str,
- method: Literal["fshift", "bshift", "nshift"] = "nshift",
- freq_check: Optional[Literal["check", "auto"]] = None,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ freq: str,
+ method: Literal["fshift", "bshift", "nshift"] = "nshift",
+ freq_check: Optional[Literal["check", "auto"]] = None,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
Function to shift data points to regular (equidistant) timestamps.
@@ -390,7 +398,7 @@ def _shift(
--------
shift : Main caller, docstring
"""
- flagged = _isflagged(flags[field], kwargs['to_mask'])
+ flagged = _isflagged(flags[field], kwargs["to_mask"])
datcol = data[field]
datcol[flagged] = np.nan
freq = evalFreqStr(freq, freq_check, datcol.index)
@@ -414,21 +422,21 @@ def _shift(
return data, flags
-@register(masking='none', module="resampling")
+@register(masking="none", module="resampling")
def resample(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- freq: str,
- agg_func: Callable[[pd.Series], pd.Series] = np.mean,
- method: Literal["fagg", "bagg", "nagg"] = "bagg",
- max_invalid_total_d: Optional[int] = None,
- max_invalid_consec_d: Optional[int] = None,
- max_invalid_consec_f: Optional[int] = None,
- max_invalid_total_f: Optional[int] = None,
- flag_agg_func: Callable[[pd.Series], float] = max,
- freq_check: Optional[Literal["check", "auto"]] = None,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ freq: str,
+ agg_func: Callable[[pd.Series], pd.Series] = np.mean,
+ method: Literal["fagg", "bagg", "nagg"] = "bagg",
+ max_invalid_total_d: Optional[int] = None,
+ max_invalid_consec_d: Optional[int] = None,
+ max_invalid_consec_f: Optional[int] = None,
+ max_invalid_total_f: Optional[int] = None,
+ flag_agg_func: Callable[[pd.Series], float] = max,
+ freq_check: Optional[Literal["check", "auto"]] = None,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
Function to resample the data. Afterwards the data will be sampled at regular (equidistant) timestamps
@@ -513,7 +521,7 @@ def resample(
The quality flags of data
Flags values and shape may have changed relatively to the flags input.
"""
- flagged = _isflagged(flags[field], kwargs['to_mask'])
+ flagged = _isflagged(flags[field], kwargs["to_mask"])
datcol = data[field]
datcol[flagged] = np.nan
freq = evalFreqStr(freq, freq_check, datcol.index)
@@ -539,9 +547,11 @@ def resample(
flags.history[field] = applyFunctionOnHistory(
flags.history[field],
- hist_func=aggregate2Freq, hist_kws=kws,
- mask_func=aggregate2Freq, mask_kws=kws,
- last_column='dummy'
+ hist_func=aggregate2Freq,
+ hist_kws=kws,
+ mask_func=aggregate2Freq,
+ mask_kws=kws,
+ last_column="dummy",
)
data[field] = datcol
@@ -549,13 +559,15 @@ def resample(
def _getChunkBounds(target: pd.Series, flagscol: pd.Series, freq: str):
- chunk_end = target.reindex(flagscol.index, method='bfill', tolerance=freq)
- chunk_start = target.reindex(flagscol.index, method='ffill', tolerance=freq)
- ignore_flags = (chunk_end.isna() | chunk_start.isna())
+ chunk_end = target.reindex(flagscol.index, method="bfill", tolerance=freq)
+ chunk_start = target.reindex(flagscol.index, method="ffill", tolerance=freq)
+ ignore_flags = chunk_end.isna() | chunk_start.isna()
return ignore_flags
-def _inverseInterpolation(source: pd.Series, target: pd.Series, freq: str, chunk_bounds) -> pd.Series:
+def _inverseInterpolation(
+ source: pd.Series, target: pd.Series, freq: str, chunk_bounds
+) -> pd.Series:
source = source.copy()
if len(chunk_bounds) > 0:
source[chunk_bounds] = np.nan
@@ -565,23 +577,29 @@ def _inverseInterpolation(source: pd.Series, target: pd.Series, freq: str, chunk
def _inverseAggregation(
- source: Union[pd.Series, pd.DataFrame],
- target: Union[pd.Series, pd.DataFrame],
- freq: str,
- method: str,
+ source: Union[pd.Series, pd.DataFrame],
+ target: Union[pd.Series, pd.DataFrame],
+ freq: str,
+ method: str,
):
return source.reindex(target.index, method=method, tolerance=freq)
-def _inverseShift(source: pd.Series, target: pd.Series, drop_mask: pd.Series,
- freq: str, method: str, fill_value) -> pd.Series:
+def _inverseShift(
+ source: pd.Series,
+ target: pd.Series,
+ drop_mask: pd.Series,
+ freq: str,
+ method: str,
+ fill_value,
+) -> pd.Series:
dtype = source.dtype
target_drops = target[drop_mask]
target = target[~drop_mask]
flags_merged = pd.merge_asof(
source,
- target.index.to_series(name='pre_index'),
+ target.index.to_series(name="pre_index"),
left_index=True,
right_index=True,
tolerance=freq,
@@ -598,18 +616,22 @@ def _inverseShift(source: pd.Series, target: pd.Series, drop_mask: pd.Series,
return source.fillna(fill_value).astype(dtype, copy=False)
-@register(masking='none', module="resampling")
+@register(masking="none", module="resampling")
def reindexFlags(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- method: Literal[
- "inverse_fagg", "inverse_bagg", "inverse_nagg",
- "inverse_fshift", "inverse_bshift", "inverse_nshift"
- ],
- source: str,
- freq: Optional[str] = None,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ method: Literal[
+ "inverse_fagg",
+ "inverse_bagg",
+ "inverse_nagg",
+ "inverse_fshift",
+ "inverse_bshift",
+ "inverse_nshift",
+ ],
+ source: str,
+ freq: Optional[str] = None,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
The Function projects flags of "source" onto flags of "field". Wherever the "field" flags are "better" then the
@@ -676,9 +698,11 @@ def reindexFlags(
if freq is None:
freq = getFreqDelta(flagscol.index)
- if freq is None and not method == 'match':
- raise ValueError('To project irregularly sampled data, either use method="match", or pass custom '
- 'projection range to freq parameter')
+ if freq is None and not method == "match":
+ raise ValueError(
+ 'To project irregularly sampled data, either use method="match", or pass custom '
+ "projection range to freq parameter"
+ )
target_datcol = data[field]
target_flagscol = flags[field]
@@ -688,7 +712,7 @@ def reindexFlags(
ignore = _getChunkBounds(target_datcol, flagscol, freq)
func = _inverseInterpolation
func_kws = dict(freq=freq, chunk_bounds=ignore, target=dummy)
- mask_kws = {**func_kws, 'chunk_bounds': []}
+ mask_kws = {**func_kws, "chunk_bounds": []}
elif method[-3:] == "agg" or method == "match":
projection_method = METHOD2ARGS[method][0]
@@ -698,17 +722,23 @@ def reindexFlags(
mask_kws = func_kws
elif method[-5:] == "shift":
- drop_mask = (target_datcol.isna() | _isflagged(target_flagscol, kwargs['to_mask']))
+ drop_mask = target_datcol.isna() | _isflagged(
+ target_flagscol, kwargs["to_mask"]
+ )
projection_method = METHOD2ARGS[method][0]
tolerance = METHOD2ARGS[method][1](freq)
func = _inverseShift
- kws = dict(freq=tolerance, method=projection_method, drop_mask=drop_mask, target=dummy)
- func_kws = {**kws, 'fill_value': UNTOUCHED}
- mask_kws = {**kws, 'fill_value': False}
+ kws = dict(
+ freq=tolerance, method=projection_method, drop_mask=drop_mask, target=dummy
+ )
+ func_kws = {**kws, "fill_value": UNTOUCHED}
+ mask_kws = {**kws, "fill_value": False}
else:
raise ValueError(f"unknown method {method}")
- history = applyFunctionOnHistory(flags.history[source], func, func_kws, func, mask_kws, last_column=dummy)
+ history = applyFunctionOnHistory(
+ flags.history[source], func, func_kws, func, mask_kws, last_column=dummy
+ )
flags.history[field] = flags.history[field].append(history, force=False)
return data, flags
diff --git a/saqc/funcs/residues.py b/saqc/funcs/residues.py
index 28a62acd1..4222416f5 100644
--- a/saqc/funcs/residues.py
+++ b/saqc/funcs/residues.py
@@ -12,18 +12,18 @@ from saqc.funcs.rolling import roll
from saqc.funcs.curvefit import fitPolynomial
-@register(masking='field', module="residues")
+@register(masking="field", module="residues")
def calculatePolynomialResidues(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- winsz: Union[str, int],
- polydeg: int,
- numba: Literal[True, False, "auto"] = "auto", # TODO: rm, not a a user decision
- eval_flags: bool = True, # TODO, not valid anymore, if still needed, maybe assign user-passed ``flag``?
- min_periods: Optional[int] = 0,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ winsz: Union[str, int],
+ polydeg: int,
+ numba: Literal[True, False, "auto"] = "auto", # TODO: rm, not a a user decision
+ eval_flags: bool = True, # TODO, not valid anymore, if still needed, maybe assign user-passed ``flag``?
+ min_periods: Optional[int] = 0,
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
Function fits a polynomial model to the data and returns the residues.
@@ -101,7 +101,9 @@ def calculatePolynomialResidues(
"""
return fitPolynomial(
- data, field, flags,
+ data,
+ field,
+ flags,
winsz=winsz,
polydeg=polydeg,
numba=numba,
@@ -113,22 +115,24 @@ def calculatePolynomialResidues(
)
-@register(masking='field', module="residues")
+@register(masking="field", module="residues")
def calculateRollingResidues(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- winsz: Union[str, int],
- func: Callable[[np.ndarray], np.ndarray] = np.mean,
- eval_flags: bool = True,
- min_periods: Optional[int] = 0,
- center: bool = True,
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ winsz: Union[str, int],
+ func: Callable[[np.ndarray], np.ndarray] = np.mean,
+ eval_flags: bool = True,
+ min_periods: Optional[int] = 0,
+ center: bool = True,
+ flag: float = BAD,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
""" TODO: docstring needed"""
return roll(
- data, field, flags,
+ data,
+ field,
+ flags,
winsz=winsz,
func=func,
eval_flags=eval_flags,
diff --git a/saqc/funcs/rolling.py b/saqc/funcs/rolling.py
index db9b026fa..918d77be5 100644
--- a/saqc/funcs/rolling.py
+++ b/saqc/funcs/rolling.py
@@ -11,19 +11,19 @@ from saqc.core import register, Flags
from saqc.lib.tools import getFreqDelta
-@register(masking='field', module="rolling")
+@register(masking="field", module="rolling")
def roll(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- winsz: Union[str, int],
- func: Callable[[pd.Series], float]=np.mean,
- eval_flags: bool=True, # TODO: not applicable anymore
- min_periods: int=0,
- center: bool=True,
- return_residues=False, # TODO: this should not be public, a wrapper would be better
- flag: float = BAD,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ winsz: Union[str, int],
+ func: Callable[[pd.Series], float] = np.mean,
+ eval_flags: bool = True, # TODO: not applicable anymore
+ min_periods: int = 0,
+ center: bool = True,
+ return_residues=False, # TODO: this should not be public, a wrapper would be better
+ flag: float = BAD,
+ **kwargs
):
"""
Models the data with the rolling mean and returns the residues.
@@ -83,25 +83,39 @@ def roll(
'sample series when rolling with "center=True".'
)
# get interval centers
- centers = np.floor((to_fit.rolling(pd.Timedelta(winsz) / 2, closed="both", min_periods=min_periods).count()))
+ centers = np.floor(
+ (
+ to_fit.rolling(
+ pd.Timedelta(winsz) / 2, closed="both", min_periods=min_periods
+ ).count()
+ )
+ )
centers = centers.drop(centers[centers.isna()].index)
centers = centers.astype(int)
- roller = to_fit.rolling(pd.Timedelta(winsz), closed="both", min_periods=min_periods)
+ roller = to_fit.rolling(
+ pd.Timedelta(winsz), closed="both", min_periods=min_periods
+ )
try:
means = getattr(roller, func.__name__)()
except AttributeError:
- means = to_fit.rolling(pd.Timedelta(winsz), closed="both", min_periods=min_periods).apply(func)
+ means = to_fit.rolling(
+ pd.Timedelta(winsz), closed="both", min_periods=min_periods
+ ).apply(func)
def center_func(x, y=centers):
pos = x.index[int(len(x) - y[x.index[-1]])]
return y.index.get_loc(pos)
- centers_iloc = centers.rolling(winsz, closed="both").apply(center_func, raw=False).astype(int)
+ centers_iloc = (
+ centers.rolling(winsz, closed="both")
+ .apply(center_func, raw=False)
+ .astype(int)
+ )
temp = means.copy()
for k in centers_iloc.iteritems():
means.iloc[k[1]] = temp[k[0]]
# last values are false, due to structural reasons:
- means[means.index[centers_iloc[-1]]: means.index[-1]] = np.nan
+ means[means.index[centers_iloc[-1]] : means.index[-1]] = np.nan
# everything is more easy if data[field] is harmonized:
else:
@@ -114,7 +128,9 @@ def roll(
try:
means = getattr(roller, func.__name__)()
except AttributeError:
- means = to_fit.rolling(window=winsz, center=center, closed="both").apply(func)
+ means = to_fit.rolling(window=winsz, center=center, closed="both").apply(
+ func
+ )
if return_residues:
means = to_fit - means
diff --git a/saqc/funcs/scores.py b/saqc/funcs/scores.py
index 02812f44a..05cdffabc 100644
--- a/saqc/funcs/scores.py
+++ b/saqc/funcs/scores.py
@@ -12,23 +12,23 @@ from saqc.lib.tools import toSequence
import saqc.lib.ts_operators as ts_ops
-@register(masking='all', module="scores")
+@register(masking="all", module="scores")
def assignKNNScore(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- fields: Sequence[str],
- n_neighbors: int = 10,
- trafo: Callable[[pd.Series], pd.Series] = lambda x: x,
- trafo_on_partition: bool = True,
- scoring_func: Callable[[pd.Series], float] = np.sum,
- target_field: str = 'kNN_scores',
- partition_freq: Union[float, str] = np.inf,
- partition_min: int = 2,
- kNN_algorithm: Literal["ball_tree", "kd_tree", "brute", "auto"] = 'ball_tree',
- metric: str = 'minkowski',
- p: int = 2,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ fields: Sequence[str],
+ n_neighbors: int = 10,
+ trafo: Callable[[pd.Series], pd.Series] = lambda x: x,
+ trafo_on_partition: bool = True,
+ scoring_func: Callable[[pd.Series], float] = np.sum,
+ target_field: str = "kNN_scores",
+ partition_freq: Union[float, str] = np.inf,
+ partition_min: int = 2,
+ kNN_algorithm: Literal["ball_tree", "kd_tree", "brute", "auto"] = "ball_tree",
+ metric: str = "minkowski",
+ p: int = 2,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
TODO: docstring need a rework
@@ -132,7 +132,9 @@ def assignKNNScore(
if isinstance(partition_freq, str):
grouper = pd.Grouper(freq=partition_freq)
else:
- grouper = pd.Series(data=np.arange(0, val_frame.shape[0]), index=val_frame.index)
+ grouper = pd.Series(
+ data=np.arange(0, val_frame.shape[0]), index=val_frame.index
+ )
grouper = grouper.transform(lambda x: int(np.floor(x / partition_freq)))
partitions = val_frame.groupby(grouper)
@@ -146,7 +148,9 @@ def assignKNNScore(
sample_size = partition.shape[0]
nn_neighbors = min(n_neighbors - 1, max(sample_size, 2))
- dist, *_ = ts_ops.kNN(partition.values, nn_neighbors, algorithm=kNN_algorithm, metric=metric, p=p)
+ dist, *_ = ts_ops.kNN(
+ partition.values, nn_neighbors, algorithm=kNN_algorithm, metric=metric, p=p
+ )
try:
resids = getattr(dist, scoring_func.__name__)(axis=1)
except AttributeError:
diff --git a/saqc/funcs/tools.py b/saqc/funcs/tools.py
index f8950debe..d07f441d3 100644
--- a/saqc/funcs/tools.py
+++ b/saqc/funcs/tools.py
@@ -11,8 +11,10 @@ from saqc.core import register, Flags
from saqc.lib.tools import periodicMask
-@register(masking='none', module="tools")
-def copy(data: DictOfSeries, field: str, flags: Flags, new_field: str, **kwargs) -> Tuple[DictOfSeries, Flags]:
+@register(masking="none", module="tools")
+def copy(
+ data: DictOfSeries, field: str, flags: Flags, new_field: str, **kwargs
+) -> Tuple[DictOfSeries, Flags]:
"""
The function generates a copy of the data "field" and inserts it under the name field + suffix into the existing
data.
@@ -46,8 +48,10 @@ def copy(data: DictOfSeries, field: str, flags: Flags, new_field: str, **kwargs)
return data, flags
-@register(masking='none', module="tools")
-def drop(data: DictOfSeries, field: str, flags: Flags, **kwargs) -> Tuple[DictOfSeries, Flags]:
+@register(masking="none", module="tools")
+def drop(
+ data: DictOfSeries, field: str, flags: Flags, **kwargs
+) -> Tuple[DictOfSeries, Flags]:
"""
The function drops field from the data dios and the flags.
@@ -74,8 +78,10 @@ def drop(data: DictOfSeries, field: str, flags: Flags, **kwargs) -> Tuple[DictOf
return data, flags
-@register(masking='none', module="tools")
-def rename(data: DictOfSeries, field: str, flags: Flags, new_name: str, **kwargs) -> Tuple[DictOfSeries, Flags]:
+@register(masking="none", module="tools")
+def rename(
+ data: DictOfSeries, field: str, flags: Flags, new_name: str, **kwargs
+) -> Tuple[DictOfSeries, Flags]:
"""
The function renames field to new name (in both, the flags and the data).
@@ -104,17 +110,17 @@ def rename(data: DictOfSeries, field: str, flags: Flags, new_name: str, **kwargs
return data, flags
-@register(masking='none', module="tools")
+@register(masking="none", module="tools")
def mask(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- mode: Literal["periodic", "mask_var"],
- mask_var: Optional[str]=None,
- period_start: Optional[str]=None,
- period_end: Optional[str]=None,
- include_bounds: bool=True,
- **kwargs,
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ mode: Literal["periodic", "mask_var"],
+ mask_var: Optional[str] = None,
+ period_start: Optional[str] = None,
+ period_end: Optional[str] = None,
+ include_bounds: bool = True,
+ **kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
This function realizes masking within saqc.
@@ -216,9 +222,9 @@ def mask(
data = data.copy()
datcol_idx = data[field].index
- if mode == 'periodic':
+ if mode == "periodic":
to_mask = periodicMask(datcol_idx, period_start, period_end, include_bounds)
- elif mode == 'mask_var':
+ elif mode == "mask_var":
idx = data[mask_var].index.intersection(datcol_idx)
to_mask = data.loc[idx, mask_var]
else:
diff --git a/saqc/funcs/transformation.py b/saqc/funcs/transformation.py
index 91952d0f1..7d4a5d75e 100644
--- a/saqc/funcs/transformation.py
+++ b/saqc/funcs/transformation.py
@@ -9,14 +9,14 @@ from dios import DictOfSeries
from saqc.core import register, Flags
-@register(masking='field', module="transformation")
+@register(masking="field", module="transformation")
def transform(
- data: DictOfSeries,
- field: str,
- flags: Flags,
- func: Callable[[pd.Series], pd.Series],
- partition_freq: Optional[Union[float, str]] = None,
- **kwargs
+ data: DictOfSeries,
+ field: str,
+ flags: Flags,
+ func: Callable[[pd.Series], pd.Series],
+ partition_freq: Optional[Union[float, str]] = None,
+ **kwargs
) -> Tuple[DictOfSeries, Flags]:
"""
Function to transform data columns with a transformation that maps series onto series of the same length.
diff --git a/saqc/lib/plotting.py b/saqc/lib/plotting.py
index 0543110c1..1ef675620 100644
--- a/saqc/lib/plotting.py
+++ b/saqc/lib/plotting.py
@@ -55,11 +55,17 @@ _plotstyle: Dict[str, dict] = {
"unflagged": dict(marker=".", ls="none", c="silver", label="UNFLAGGED"),
"good": dict(marker=".", fillstyle="none", ls="none", c="seagreen", label="GOOD"),
"bad": dict(marker=".", fillstyle="none", ls="none", c="firebrick", label="BAD"),
- "suspicious": dict(marker=".", fillstyle="none", ls="none", c="gold", label="SUSPICIOUS"),
- "old-flags": dict(marker=".", fillstyle="none", ls="none", c="black", label="old-flags"),
+ "suspicious": dict(
+ marker=".", fillstyle="none", ls="none", c="gold", label="SUSPICIOUS"
+ ),
+ "old-flags": dict(
+ marker=".", fillstyle="none", ls="none", c="black", label="old-flags"
+ ),
# data
"data": dict(c="silver", ls="-", label="data"),
- "data-nans": dict(marker=".", fillstyle="none", ls="none", c="lightsteelblue", label="NaN"),
+ "data-nans": dict(
+ marker=".", fillstyle="none", ls="none", c="lightsteelblue", label="NaN"
+ ),
}
@@ -69,7 +75,11 @@ def _show():
def plotAllHook(
- data, flagger, targets=None, show_info_table: bool = True, annotations: Optional[dios.DictOfSeries] = None,
+ data,
+ flagger,
+ targets=None,
+ show_info_table: bool = True,
+ annotations: Optional[dios.DictOfSeries] = None,
):
__importHelper()
targets = flagger.columns if targets is None else targets
@@ -110,7 +120,9 @@ def plotHook(
)
if len(targets) == 1:
- _plotSingleVariable(**args, sources=sources, show_reference_data=True, plot_name=plot_name)
+ _plotSingleVariable(
+ **args, sources=sources, show_reference_data=True, plot_name=plot_name
+ )
else:
_plotMultipleVariables(**args)
@@ -171,7 +183,7 @@ def _plotMultipleVariables(
sharex=True,
tight_layout=True,
squeeze=False,
- gridspec_kw=gs_kw if show_tab else {}
+ gridspec_kw=gs_kw if show_tab else {},
)
# plot max. 4 plots per figure
@@ -303,7 +315,10 @@ def _plotSingleVariable(
logging.warning(f"plotting: only first 4 of {slen} sources are shown.")
slen = 4
- fig = plt.figure(constrained_layout=True, figsize=_figsize,)
+ fig = plt.figure(
+ constrained_layout=True,
+ figsize=_figsize,
+ )
outer_gs = fig.add_gridspec(ncols=1, nrows=nrows)
gs_count = 0
allaxs = []
@@ -422,7 +437,9 @@ def _getDataFromVar(
# though the calculations would work.
if flags_old.index.equals(flags_new.index):
unchanged, changed = _splitOldAndNew(flags_old, flags_new)
- unchanged, changed = _projectFlagsOntoData([unchanged, changed], plotdict["data"])
+ unchanged, changed = _projectFlagsOntoData(
+ [unchanged, changed], plotdict["data"]
+ )
plotdict["unchanged"] = unchanged
plotdict["changed"] = changed
@@ -601,7 +618,9 @@ def _plotDataWithTable(fig, gs, pdict, show_tab=True):
_plotInfoTable()
"""
if show_tab:
- plot_gs, tab_gs = gs.subgridspec(ncols=2, nrows=1, width_ratios=_layout_data_to_table_ratio)
+ plot_gs, tab_gs = gs.subgridspec(
+ ncols=2, nrows=1, width_ratios=_layout_data_to_table_ratio
+ )
ax = fig.add_subplot(tab_gs)
_plotInfoTable(ax, pdict, _plotstyle, len(pdict["data"]))
ax = fig.add_subplot(plot_gs)
@@ -648,9 +667,9 @@ def _plotFromDicts(ax, plotdict, styledict):
def _annotate(ax, plotdict, txtseries: pd.Series):
for x, txt in txtseries.iteritems():
try:
- y = plotdict['data'].loc[x]
+ y = plotdict["data"].loc[x]
if np.isnan(y):
- y = plotdict['data-nans'].loc[x]
+ y = plotdict["data-nans"].loc[x]
except KeyError:
continue
ax.annotate(txt, xy=(x, y), rotation=45)
diff --git a/saqc/lib/rolling.py b/saqc/lib/rolling.py
index 4af3e7cb4..abce27bfc 100644
--- a/saqc/lib/rolling.py
+++ b/saqc/lib/rolling.py
@@ -30,14 +30,23 @@ from pandas.core.window.indexers import calculate_variable_window_bounds
from pandas.core.window.rolling import Rolling, Window
-def is_slice(k): return isinstance(k, slice)
+def is_slice(k):
+ return isinstance(k, slice)
class _CustomBaseIndexer(BaseIndexer):
is_datetimelike = None
- def __init__(self, index_array, window_size, center=False, forward=False,
- expand=False, step=None, mask=None):
+ def __init__(
+ self,
+ index_array,
+ window_size,
+ center=False,
+ forward=False,
+ expand=False,
+ step=None,
+ mask=None,
+ ):
super().__init__()
self.index_array = index_array
self.window_size = window_size
@@ -61,19 +70,21 @@ class _CustomBaseIndexer(BaseIndexer):
if is_integer(self.step) or self.step is None:
self.step = slice(None, None, self.step or None)
if not is_slice(self.step):
- raise TypeError('step must be integer or slice.')
+ raise TypeError("step must be integer or slice.")
if self.step == slice(None):
self.step = None
if self.skip is not None:
if len(self.index_array) != len(self.skip):
- raise ValueError('mask must have same length as data to roll over.')
+ raise ValueError("mask must have same length as data to roll over.")
self.skip = np.array(self.skip)
if self.skip.dtype != bool:
- raise TypeError('mask must have boolean values only.')
+ raise TypeError("mask must have boolean values only.")
self.skip = ~self.skip
- def get_window_bounds(self, num_values=0, min_periods=None, center=None, closed=None):
+ def get_window_bounds(
+ self, num_values=0, min_periods=None, center=None, closed=None
+ ):
if min_periods is None:
assert self.is_datetimelike is False
min_periods = 1
@@ -188,7 +199,7 @@ class _FixedWindowDirectionIndexer(_CustomBaseIndexer):
def _bw(self, num_values=0, offset=0):
start = np.arange(-self.window_size, num_values + offset, dtype="int64") + 1
end = start + self.window_size
- start[:self.window_size] = 0
+ start[: self.window_size] = 0
return start, end
def _fw(self, num_values=0, offset=0):
@@ -209,8 +220,8 @@ class _VariableWindowDirectionIndexer(_CustomBaseIndexer):
ws_bw, ws_fw = self._get_center_window_sizes(center, self.window_size)
if center:
c1 = c2 = closed
- if closed == 'neither':
- c1, c2 = 'right', 'left'
+ if closed == "neither":
+ c1, c2 = "right", "left"
start, _ = self._bw(num_values, ws_bw, c1)
_, end = self._fw(num_values, ws_fw, c2)
@@ -248,23 +259,38 @@ class _VariableWindowDirectionIndexer(_CustomBaseIndexer):
def _bw(self, num_values, window_size, closed):
arr = self.index_array
- start, end = calculate_variable_window_bounds(num_values, window_size, None, None, closed, arr)
+ start, end = calculate_variable_window_bounds(
+ num_values, window_size, None, None, closed, arr
+ )
return start, end
def _fw(self, num_values, window_size, closed):
arr = self.index_array[::-1]
- s, _ = calculate_variable_window_bounds(num_values, window_size, None, None, closed, arr)
+ s, _ = calculate_variable_window_bounds(
+ num_values, window_size, None, None, closed, arr
+ )
start = np.arange(num_values)
end = num_values - s[::-1]
- if closed in ['left', 'neither']:
+ if closed in ["left", "neither"]:
start += 1
return start, end
-def customRoller(obj, window, min_periods=None, # aka minimum non-nan values
- center=False, win_type=None, on=None, axis=0, closed=None,
- forward=False, expand=True, step=None, mask=None) -> Union[Rolling, Window]:
+def customRoller(
+ obj,
+ window,
+ min_periods=None, # aka minimum non-nan values
+ center=False,
+ win_type=None,
+ on=None,
+ axis=0,
+ closed=None,
+ forward=False,
+ expand=True,
+ step=None,
+ mask=None,
+) -> Union[Rolling, Window]:
"""
A custom rolling implementation, using pandas as base.
@@ -294,7 +320,7 @@ def customRoller(obj, window, min_periods=None, # aka minimum non-nan values
For a DataFrame, a datetime-like column or MultiIndex level on which to calculate the rolling window,
rather than the DataFrame’s index. Provided integer column is ignored and excluded from result since
an integer index is not used to calculate the rolling window.
-
+
axis : int or str, default 0
closed : str, default None
@@ -346,7 +372,9 @@ def customRoller(obj, window, min_periods=None, # aka minimum non-nan values
# center is the only param from the pandas rolling implementation
# that we advance, namely we allow center=True on dt-indexed data
# that's why we take it as ours
- theirs = dict(min_periods=min_periods, win_type=win_type, on=on, axis=axis, closed=closed)
+ theirs = dict(
+ min_periods=min_periods, win_type=win_type, on=on, axis=axis, closed=closed
+ )
ours = dict(center=center, forward=forward, expand=expand, step=step, mask=mask)
assert len(theirs) + len(ours) == num_params, "not all params covert (!)"
@@ -359,7 +387,11 @@ def customRoller(obj, window, min_periods=None, # aka minimum non-nan values
except Exception:
raise
- indexer = _VariableWindowDirectionIndexer if x.is_freq_type else _FixedWindowDirectionIndexer
+ indexer = (
+ _VariableWindowDirectionIndexer
+ if x.is_freq_type
+ else _FixedWindowDirectionIndexer
+ )
indexer = indexer(index_array=x._on.asi8, window_size=x.window, **ours)
# Centering is fully done in our own indexers. So we do not pass center to rolling(). Especially because
diff --git a/saqc/lib/tools.py b/saqc/lib/tools.py
index 1f15df6d0..6d9d5d2de 100644
--- a/saqc/lib/tools.py
+++ b/saqc/lib/tools.py
@@ -30,7 +30,9 @@ def assertScalar(name, value, optional=False):
raise ValueError(f"'{name}' needs to be a scalar")
-def toSequence(value: Union[T, Sequence[T]], default: Union[T, Sequence[T]] = None) -> Sequence[T]:
+def toSequence(
+ value: Union[T, Sequence[T]], default: Union[T, Sequence[T]] = None
+) -> Sequence[T]:
if value is None:
value = default
if np.isscalar(value):
@@ -199,6 +201,7 @@ def periodicMask(dtindex, season_start, season_end, include_bounds):
When inclusive_selection="season", all above examples work the same way, only that you now
determine wich values NOT TO mask (=wich values are to constitute the "seasons").
"""
+
def _replaceBuilder(stamp):
keys = ("second", "minute", "hour", "day", "month", "year")
stamp_list = map(int, re.split(r"[-T:]", stamp)[::-1])
@@ -219,22 +222,27 @@ def periodicMask(dtindex, season_start, season_end, include_bounds):
end_replacer = _replaceBuilder(season_end)
if pd.Timestamp(start_replacer(dtindex)) <= pd.Timestamp(end_replacer(dtindex)):
+
def _selector(x, base_bool=include_bounds):
- x[start_replacer(x.index):end_replacer(x.index)] = not base_bool
+ x[start_replacer(x.index) : end_replacer(x.index)] = not base_bool
return x
+
else:
+
def _selector(x, base_bool=include_bounds):
- x[:end_replacer(x.index)] = not base_bool
- x[start_replacer(x.index):] = not base_bool
+ x[: end_replacer(x.index)] = not base_bool
+ x[start_replacer(x.index) :] = not base_bool
return x
- freq = '1' + 'mmmhhhdddMMMYYY'[len(season_start)]
+ freq = "1" + "mmmhhhdddMMMYYY"[len(season_start)]
return mask.groupby(pd.Grouper(freq=freq)).transform(_selector)
def assertDictOfSeries(df: Any, argname: str = "arg") -> None:
if not isinstance(df, dios.DictOfSeries):
- raise TypeError(f"{argname} must be of type dios.DictOfSeries, {type(df)} was given")
+ raise TypeError(
+ f"{argname} must be of type dios.DictOfSeries, {type(df)} was given"
+ )
def assertSeries(srs: Any, argname: str = "arg") -> None:
@@ -315,8 +323,16 @@ def mutateIndex(index, old_name, new_name):
return index
-def estimateFrequency(index, delta_precision=-1, max_rate="10s", min_rate="1D", optimize=True,
- min_energy=0.2, max_freqs=10, bins=None):
+def estimateFrequency(
+ index,
+ delta_precision=-1,
+ max_rate="10s",
+ min_rate="1D",
+ optimize=True,
+ min_energy=0.2,
+ max_freqs=10,
+ bins=None,
+):
"""
Function to estimate the sampling rate of an index.
@@ -371,23 +387,27 @@ def estimateFrequency(index, delta_precision=-1, max_rate="10s", min_rate="1D",
"""
index_n = index.to_numpy(float)
if index.empty:
- return 'empty', []
+ return "empty", []
- index_n = (index_n - index_n[0])*10**(-9 + delta_precision)
- delta = np.zeros(int(index_n[-1])+1)
+ index_n = (index_n - index_n[0]) * 10 ** (-9 + delta_precision)
+ delta = np.zeros(int(index_n[-1]) + 1)
delta[index_n.astype(int)] = 1
if optimize:
delta_f = np.abs(fft.rfft(delta, fft.next_fast_len(len(delta))))
else:
delta_f = np.abs(fft.rfft(delta))
- len_f = len(delta_f)*2
- min_energy = delta_f[0]*min_energy
+ len_f = len(delta_f) * 2
+ min_energy = delta_f[0] * min_energy
# calc/assign low/high freq cut offs (makes life easier):
- min_rate_i = int(len_f/(pd.Timedelta(min_rate).total_seconds()*(10**delta_precision)))
+ min_rate_i = int(
+ len_f / (pd.Timedelta(min_rate).total_seconds() * (10 ** delta_precision))
+ )
delta_f[:min_rate_i] = 0
- max_rate_i = int(len_f/(pd.Timedelta(max_rate).total_seconds()*(10**delta_precision)))
- hf_cutoff = min(max_rate_i, len_f//2)
+ max_rate_i = int(
+ len_f / (pd.Timedelta(max_rate).total_seconds() * (10 ** delta_precision))
+ )
+ hf_cutoff = min(max_rate_i, len_f // 2)
delta_f[hf_cutoff:] = 0
delta_f[delta_f < min_energy] = 0
@@ -395,54 +415,66 @@ def estimateFrequency(index, delta_precision=-1, max_rate="10s", min_rate="1D",
freqs = []
f_i = np.argmax(delta_f)
while (f_i > 0) & (len(freqs) < max_freqs):
- f = (len_f / f_i)/(60*10**(delta_precision))
+ f = (len_f / f_i) / (60 * 10 ** (delta_precision))
freqs.append(f)
- for i in range(1, hf_cutoff//f_i + 1):
- delta_f[(i*f_i) - min_rate_i:(i*f_i) + min_rate_i] = 0
+ for i in range(1, hf_cutoff // f_i + 1):
+ delta_f[(i * f_i) - min_rate_i : (i * f_i) + min_rate_i] = 0
f_i = np.argmax(delta_f)
if len(freqs) == 0:
return None, []
if bins is None:
- r = range(0, int(pd.Timedelta(min_rate).total_seconds()/60))
+ r = range(0, int(pd.Timedelta(min_rate).total_seconds() / 60))
bins = [0, 0.1, 0.2, 0.3, 0.4] + [i + 0.5 for i in r]
f_hist, bins = np.histogram(freqs, bins=bins)
freqs = np.ceil(bins[:-1][f_hist >= 1])
- gcd_freq = np.gcd.reduce((10*freqs).astype(int))/10
+ gcd_freq = np.gcd.reduce((10 * freqs).astype(int)) / 10
- return str(int(gcd_freq)) + 'min', [str(int(i)) + 'min' for i in freqs]
+ return str(int(gcd_freq)) + "min", [str(int(i)) + "min" for i in freqs]
def evalFreqStr(freq, check, index):
- if check in ['check', 'auto']:
+ if check in ["check", "auto"]:
f_passed = freq
freq = index.inferred_freq
freqs = [freq]
if freq is None:
freq, freqs = estimateFrequency(index)
if freq is None:
- logging.warning('Sampling rate could not be estimated.')
+ logging.warning("Sampling rate could not be estimated.")
if len(freqs) > 1:
- logging.warning(f"Sampling rate seems to be not uniform!."
- f"Detected: {freqs}")
+ logging.warning(
+ f"Sampling rate seems to be not uniform!." f"Detected: {freqs}"
+ )
- if check == 'check':
+ if check == "check":
f_passed_seconds = pd.Timedelta(f_passed).total_seconds()
freq_seconds = pd.Timedelta(freq).total_seconds()
if f_passed_seconds != freq_seconds:
- logging.warning(f"Sampling rate estimate ({freq}) missmatches passed frequency ({f_passed}).")
- elif check == 'auto':
+ logging.warning(
+ f"Sampling rate estimate ({freq}) missmatches passed frequency ({f_passed})."
+ )
+ elif check == "auto":
if freq is None:
- raise ValueError('Frequency estimation for non-empty series failed with no fall back frequency passed.')
+ raise ValueError(
+ "Frequency estimation for non-empty series failed with no fall back frequency passed."
+ )
f_passed = freq
else:
f_passed = freq
return f_passed
-def detectDeviants(data, metric, norm_spread, norm_frac, linkage_method='single', population='variables'):
+def detectDeviants(
+ data,
+ metric,
+ norm_spread,
+ norm_frac,
+ linkage_method="single",
+ population="variables",
+):
"""
Helper function for carrying out the repeatedly upcoming task,
of detecting variables a group of variables.
@@ -491,17 +523,19 @@ def detectDeviants(data, metric, norm_spread, norm_frac, linkage_method='single'
condensed = np.abs(dist_mat[tuple(zip(*combs))])
Z = linkage(condensed, method=linkage_method)
- cluster = fcluster(Z, norm_spread, criterion='distance')
- if population == 'variables':
+ cluster = fcluster(Z, norm_spread, criterion="distance")
+ if population == "variables":
counts = collections.Counter(cluster)
pop_num = var_num
- elif population == 'samples':
- counts = {cluster[j]: 0 for j in range(0,var_num)}
+ elif population == "samples":
+ counts = {cluster[j]: 0 for j in range(0, var_num)}
for c in range(var_num):
counts[cluster[c]] += data.iloc[:, c].dropna().shape[0]
pop_num = np.sum(list(counts.values()))
else:
- raise ValueError("Not a valid normality criteria keyword passed. Pass either 'variables' or 'population'.")
+ raise ValueError(
+ "Not a valid normality criteria keyword passed. Pass either 'variables' or 'population'."
+ )
norm_cluster = -1
for item in counts.items():
@@ -526,7 +560,7 @@ def getFreqDelta(index):
(``None`` will also be returned for pd.RangeIndex type.)
"""
- delta = getattr(index, 'freq', None)
+ delta = getattr(index, "freq", None)
if delta is None and not index.empty:
i = pd.date_range(index[0], index[-1], len(index))
if i.equals(index):
diff --git a/saqc/lib/ts_operators.py b/saqc/lib/ts_operators.py
index 1904f6875..923c5641f 100644
--- a/saqc/lib/ts_operators.py
+++ b/saqc/lib/ts_operators.py
@@ -59,7 +59,10 @@ def derivative(ts, unit="1min"):
def deltaT(ts, unit="1min"):
# calculates series of time gaps in ts
- return ts.index.to_series().diff().dt.total_seconds() / pd.Timedelta(unit).total_seconds()
+ return (
+ ts.index.to_series().diff().dt.total_seconds()
+ / pd.Timedelta(unit).total_seconds()
+ )
def difference(ts):
@@ -111,11 +114,12 @@ def standardizeByIQR(ts):
return (ts - np.median(ts)) / iqr(ts, nan_policy="omit")
-def kNN(in_arr, n_neighbors, algorithm="ball_tree", metric='minkowski', p=2):
+def kNN(in_arr, n_neighbors, algorithm="ball_tree", metric="minkowski", p=2):
# k-nearest-neighbor search
- nbrs = NearestNeighbors(n_neighbors=n_neighbors, algorithm=algorithm, metric=metric, p=p)\
- .fit(in_arr.reshape(in_arr.shape[0], -1))
+ nbrs = NearestNeighbors(
+ n_neighbors=n_neighbors, algorithm=algorithm, metric=metric, p=p
+ ).fit(in_arr.reshape(in_arr.shape[0], -1))
return nbrs.kneighbors()
@@ -166,18 +170,26 @@ def validationTrafo(data, max_nan_total, max_nan_consec):
def stdQC(data, max_nan_total=np.inf, max_nan_consec=np.inf):
- return np.nanstd(data[~validationTrafo(data.isna(), max_nan_total, max_nan_consec)], ddof=1)
+ return np.nanstd(
+ data[~validationTrafo(data.isna(), max_nan_total, max_nan_consec)], ddof=1
+ )
def varQC(data, max_nan_total=np.inf, max_nan_consec=np.inf):
- return np.nanvar(data[~validationTrafo(data.isna(), max_nan_total, max_nan_consec)], ddof=1)
+ return np.nanvar(
+ data[~validationTrafo(data.isna(), max_nan_total, max_nan_consec)], ddof=1
+ )
def meanQC(data, max_nan_total=np.inf, max_nan_consec=np.inf):
- return np.nanmean(data[~validationTrafo(data.isna(), max_nan_total, max_nan_consec)])
+ return np.nanmean(
+ data[~validationTrafo(data.isna(), max_nan_total, max_nan_consec)]
+ )
-def interpolateNANs(data, method, order=2, inter_limit=2, downgrade_interpolation=False):
+def interpolateNANs(
+ data, method, order=2, inter_limit=2, downgrade_interpolation=False
+):
"""
The function interpolates nan-values (and nan-grids) in timeseries data. It can be passed all the method keywords
from the pd.Series.interpolate method and will than apply this very methods. Note, that the inter_limit keyword
@@ -207,7 +219,10 @@ def interpolateNANs(data, method, order=2, inter_limit=2, downgrade_interpolatio
gap_mask = gap_mask & gap_mask.shift(-1, fill_value=True)
else:
gap_mask = (
- gap_mask.replace(True, np.nan).fillna(method="bfill", limit=inter_limit).replace(np.nan, True).astype(bool)
+ gap_mask.replace(True, np.nan)
+ .fillna(method="bfill", limit=inter_limit)
+ .replace(np.nan, True)
+ .astype(bool)
)
pre_index = data.index
@@ -215,7 +230,9 @@ def interpolateNANs(data, method, order=2, inter_limit=2, downgrade_interpolatio
if method in ["linear", "time"]:
- data.interpolate(method=method, inplace=True, limit=inter_limit - 1, limit_area="inside")
+ data.interpolate(
+ method=method, inplace=True, limit=inter_limit - 1, limit_area="inside"
+ )
else:
dat_name = data.name
@@ -250,7 +267,13 @@ def interpolateNANs(data, method, order=2, inter_limit=2, downgrade_interpolatio
def aggregate2Freq(
- data: pd.Series, method, freq, agg_func, fill_value=np.nan, max_invalid_total=None, max_invalid_consec=None
+ data: pd.Series,
+ method,
+ freq,
+ agg_func,
+ fill_value=np.nan,
+ max_invalid_total=None,
+ max_invalid_consec=None,
):
"""
The function aggregates values to an equidistant frequency grid with agg_func.
@@ -258,7 +281,7 @@ def aggregate2Freq(
also serves as a replacement for "invalid" intervals.
"""
methods = {
- "nagg": lambda seconds_total: (seconds_total/2, "left", "left"),
+ "nagg": lambda seconds_total: (seconds_total / 2, "left", "left"),
"bagg": lambda _: (0, "left", "left"),
"fagg": lambda _: (0, "right", "right"),
}
@@ -271,7 +294,9 @@ def aggregate2Freq(
temp_mask = data == fill_value
temp_mask = temp_mask.groupby(pd.Grouper(freq=freq)).transform(
- validationTrafo, max_nan_total=max_invalid_total, max_nan_consec=max_invalid_consec
+ validationTrafo,
+ max_nan_total=max_invalid_total,
+ max_nan_consec=max_invalid_consec,
)
data[temp_mask] = fill_value
@@ -284,7 +309,9 @@ def aggregate2Freq(
# - we are aggregating data and flags with this function and empty intervals usually would get assigned BAD
# flag (where resample inserts np.nan or 0)
- data_resampler = data.resample(f"{seconds_total:.0f}s", base=base, closed=closed, label=label)
+ data_resampler = data.resample(
+ f"{seconds_total:.0f}s", base=base, closed=closed, label=label
+ )
empty_intervals = data_resampler.count() == 0
# great performance gain can be achieved, when avoiding .apply and using pd.resampler
@@ -292,7 +319,7 @@ def aggregate2Freq(
try:
check_name = re.sub("^nan", "", agg_func.__name__)
# a nasty special case: if function "count" was passed, we not want empty intervals to be replaced by nan:
- if check_name == 'count':
+ if check_name == "count":
empty_intervals[:] = False
data = getattr(data_resampler, check_name)()
except AttributeError:
@@ -308,7 +335,9 @@ def aggregate2Freq(
return data
-def shift2Freq(data: Union[pd.Series, pd.DataFrame], method: str, freq: str, fill_value):
+def shift2Freq(
+ data: Union[pd.Series, pd.DataFrame], method: str, freq: str, fill_value
+):
"""
shift timestamps backwards/forwards in order to align them with an equidistant
frequency grid. Resulting Nan's are replaced with the fill-value.
@@ -317,15 +346,18 @@ def shift2Freq(data: Union[pd.Series, pd.DataFrame], method: str, freq: str, fil
methods = {
"fshift": lambda freq: ("ffill", pd.Timedelta(freq)),
"bshift": lambda freq: ("bfill", pd.Timedelta(freq)),
- "nshift": lambda freq: ("nearest", pd.Timedelta(freq)/2),
+ "nshift": lambda freq: ("nearest", pd.Timedelta(freq) / 2),
}
direction, tolerance = methods[method](freq)
target_ind = pd.date_range(
- start=data.index[0].floor(freq), end=data.index[-1].ceil(freq),
+ start=data.index[0].floor(freq),
+ end=data.index[-1].ceil(freq),
freq=freq,
- name=data.index.name
+ name=data.index.name,
+ )
+ return data.reindex(
+ target_ind, method=direction, tolerance=tolerance, fill_value=fill_value
)
- return data.reindex(target_ind, method=direction, tolerance=tolerance, fill_value=fill_value)
@nb.njit
@@ -420,7 +452,7 @@ def expDriftModel(x, c, origin=None, target=None):
def linearDriftModel(x, origin=None, target=None):
- return origin + x*target
+ return origin + x * target
def linearInterpolation(data, inter_limit=2):
@@ -428,4 +460,6 @@ def linearInterpolation(data, inter_limit=2):
def polynomialInterpolation(data, inter_limit=2, inter_order=2):
- return interpolateNANs(data, "polynomial", inter_limit=inter_limit, order=inter_order)
+ return interpolateNANs(
+ data, "polynomial", inter_limit=inter_limit, order=inter_order
+ )
diff --git a/saqc/lib/types.py b/saqc/lib/types.py
index fd8352fc1..18b2fbeb1 100644
--- a/saqc/lib/types.py
+++ b/saqc/lib/types.py
@@ -1,19 +1,19 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
__all__ = [
- 'T',
- 'ArrayLike',
- 'PandasLike',
- 'DiosLikeT',
- 'FuncReturnT',
- 'FreqString',
- 'ColumnName',
- 'IntegerWindow',
- 'TimestampColumnName',
- 'CurveFitter',
+ "T",
+ "ArrayLike",
+ "PandasLike",
+ "DiosLikeT",
+ "FuncReturnT",
+ "FreqString",
+ "ColumnName",
+ "IntegerWindow",
+ "TimestampColumnName",
+ "CurveFitter",
"ExternalFlag",
"CallStack",
- "CalledStack"
+ "CalledStack",
]
from typing import TypeVar, Union, NewType, List, Tuple, Optional
@@ -35,7 +35,9 @@ FuncReturnT = Tuple[DictOfSeries, Flags]
ExternalFlag = Union[str, float, int]
# we only support fixed length offsets
-FreqString = NewType("FreqString", Literal["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"])
+FreqString = NewType(
+ "FreqString", Literal["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"]
+)
CallGraph = List[Tuple[ColumnSelector, APIController, SaQCFunction]]
MaterializedGraph = List[Tuple[ColumnSelector, Optional[SaQCFunction]]]
diff --git a/tests/common.py b/tests/common.py
index 21fc6c9c2..162da2bb1 100644
--- a/tests/common.py
+++ b/tests/common.py
@@ -20,7 +20,9 @@ def flagAll(data, field, flags, **kwargs):
return data, flags
-def initData(cols=2, start_date="2017-01-01", end_date="2017-12-31", freq=None, rows=None):
+def initData(
+ cols=2, start_date="2017-01-01", end_date="2017-12-31", freq=None, rows=None
+):
if rows is None:
freq = freq or "1h"
@@ -85,5 +87,3 @@ def checkDataFlagsInvariants(data, flags, field, identical=True):
assert data[field].index.identical(flags[field].index)
else:
assert data[field].index.equals(flags[field].index)
-
-
diff --git a/tests/core/test_core.py b/tests/core/test_core.py
index 5370f520d..7ca08c680 100644
--- a/tests/core/test_core.py
+++ b/tests/core/test_core.py
@@ -22,7 +22,7 @@ logging.disable(logging.CRITICAL)
OPTIONAL = [False, True]
-register(masking='field')(flagAll)
+register(masking="field")(flagAll)
@pytest.fixture
@@ -37,8 +37,7 @@ def flags(data, optional):
def test_errorHandling(data):
-
- @register(masking='field')
+ @register(masking="field")
def raisingFunc(data, field, flags, **kwargs):
raise TypeError
@@ -49,7 +48,7 @@ def test_errorHandling(data):
SaQC(data, error_policy=policy).raisingFunc(var1).getResult()
with pytest.raises(TypeError):
- SaQC(data, error_policy='raise').raisingFunc(var1).getResult()
+ SaQC(data, error_policy="raise").raisingFunc(var1).getResult()
def test_duplicatedVariable():
@@ -89,7 +88,9 @@ def test_dtypes(data, flags):
flags_raw = flags.toDios()
var1, var2 = data.columns[:2]
- pdata, pflags = SaQC(data, flags=flags_raw).flagAll(var1).flagAll(var2).getResult(raw=True)
+ pdata, pflags = (
+ SaQC(data, flags=flags_raw).flagAll(var1).flagAll(var2).getResult(raw=True)
+ )
for c in pflags.columns:
assert pflags[c].dtype == flags[c].dtype
@@ -106,8 +107,14 @@ def test_plotting(data):
field, *_ = data.columns
flags = initFlagsLike(data)
_, flags_range = flagRange(data, field, flags, min=10, max=90, flag=BAD)
- data_new, flags_range = flagRange(data, field, flags_range, min=40, max=60, flag=DOUBT)
+ data_new, flags_range = flagRange(
+ data, field, flags_range, min=40, max=60, flag=DOUBT
+ )
splot._interactive = False
- splot._plotSingleVariable(data, data_new, flags, flags_range, sources=[], targets=[data_new.columns[0]])
- splot._plotMultipleVariables(data, data_new, flags, flags_range, targets=data_new.columns)
+ splot._plotSingleVariable(
+ data, data_new, flags, flags_range, sources=[], targets=[data_new.columns[0]]
+ )
+ splot._plotMultipleVariables(
+ data, data_new, flags, flags_range, targets=data_new.columns
+ )
splot._interactive = True
diff --git a/tests/core/test_creation.py b/tests/core/test_creation.py
index b9b931d29..9d2badf6d 100644
--- a/tests/core/test_creation.py
+++ b/tests/core/test_creation.py
@@ -8,11 +8,13 @@ import dios
def test_init():
from saqc import SaQC, Flags
- arr = np.array([
- [0, 1, 2],
- [0, 1, 3],
- ])
- data = pd.DataFrame(arr, columns=list('abc'))
+ arr = np.array(
+ [
+ [0, 1, 2],
+ [0, 1, 3],
+ ]
+ )
+ data = pd.DataFrame(arr, columns=list("abc"))
qc = SaQC(data)
assert isinstance(qc, SaQC)
diff --git a/tests/core/test_flags.py b/tests/core/test_flags.py
index 8877cae6b..584999bc9 100644
--- a/tests/core/test_flags.py
+++ b/tests/core/test_flags.py
@@ -14,29 +14,30 @@ from tests.core.test_history import (
)
_data = [
-
np.array([[]]),
np.zeros((1, 1)),
np.zeros((3, 4)),
np.ones((3, 4)),
np.ones((3, 4)) * np.nan,
-
- np.array([
- [0, 0, 0, 0],
- [0, 1, 2, 3],
- [0, 1, 2, 3],
- ]),
-
- np.array([
- [0, 0, 0, 0],
- [0, 1, np.nan, 3],
- [0, 1, 2, 3],
- ]),
+ np.array(
+ [
+ [0, 0, 0, 0],
+ [0, 1, 2, 3],
+ [0, 1, 2, 3],
+ ]
+ ),
+ np.array(
+ [
+ [0, 0, 0, 0],
+ [0, 1, np.nan, 3],
+ [0, 1, 2, 3],
+ ]
+ ),
]
data = []
for d in _data:
- columns = list('abcdefgh')[:d.shape[1]]
+ columns = list("abcdefgh")[: d.shape[1]]
df = pd.DataFrame(d, dtype=float, columns=columns)
dis = dios.DictOfSeries(df)
di = {}
@@ -46,7 +47,7 @@ for d in _data:
data.append(dis)
-@pytest.mark.parametrize('data', data)
+@pytest.mark.parametrize("data", data)
def test_init(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
flags = Flags(data)
assert isinstance(flags, Flags)
@@ -59,7 +60,7 @@ def is_equal(f1, f2):
assert hist_equal(f1.history[c], f2.history[c])
-@pytest.mark.parametrize('data', data)
+@pytest.mark.parametrize("data", data)
def test_copy(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
flags = Flags(data)
shallow = flags.copy(deep=False)
@@ -83,8 +84,10 @@ def test_copy(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]
assert deep._data[c] is not flags._data[c]
-@pytest.mark.parametrize('data', data)
-def test_flags_history(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
+@pytest.mark.parametrize("data", data)
+def test_flags_history(
+ data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]
+):
flags = Flags(data)
# get
@@ -97,13 +100,13 @@ def test_flags_history(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd
for c in flags.columns:
hist = flags.history[c]
hlen = len(hist)
- hist.append(pd.Series(888., index=hist.index, dtype=float))
+ hist.append(pd.Series(888.0, index=hist.index, dtype=float))
flags.history[c] = hist
assert isinstance(hist, History)
assert len(hist) == hlen + 1
-@pytest.mark.parametrize('data', data)
+@pytest.mark.parametrize("data", data)
def test_get_flags(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
flags = Flags(data)
@@ -118,70 +121,72 @@ def test_get_flags(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Ser
assert var is not flags[c]
# in particular, a deep copy
- var[:] = 9999.
+ var[:] = 9999.0
assert all(flags[c] != var)
-@pytest.mark.parametrize('data', data)
+@pytest.mark.parametrize("data", data)
def test_set_flags(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
flags = Flags(data)
for c in flags.columns:
var = flags[c]
hlen = len(flags.history[c])
- new = pd.Series(9999., index=var.index, dtype=float)
+ new = pd.Series(9999.0, index=var.index, dtype=float)
flags[c] = new
assert len(flags.history[c]) == hlen + 1
- assert all(flags.history[c].max() == 9999.)
+ assert all(flags.history[c].max() == 9999.0)
assert all(flags.history[c].max() == flags[c])
# check if deep-copied correctly
- new[:] = 8888.
- assert all(flags.history[c].max() == 9999.)
+ new[:] = 8888.0
+ assert all(flags.history[c].max() == 9999.0)
# flags always overwrite former
flags[c] = new
assert len(flags.history[c]) == hlen + 2
- assert all(flags.history[c].max() == 8888.)
+ assert all(flags.history[c].max() == 8888.0)
assert all(flags.history[c].max() == flags[c])
# check if deep-copied correctly
- new[:] = 7777.
- assert all(flags.history[c].max() == 8888.)
+ new[:] = 7777.0
+ assert all(flags.history[c].max() == 8888.0)
-@pytest.mark.parametrize('data', data)
-def test_set_flags_with_mask(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
+@pytest.mark.parametrize("data", data)
+def test_set_flags_with_mask(
+ data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]
+):
flags = Flags(data)
for c in flags.columns:
var = flags[c]
mask = var == UNFLAGGED
- scalar = 222.
+ scalar = 222.0
flags[mask, c] = scalar
- assert all(flags[c].loc[mask] == 222.)
- assert all(flags[c].loc[~mask] != 222.)
+ assert all(flags[c].loc[mask] == 222.0)
+ assert all(flags[c].loc[~mask] != 222.0)
# scalar without mask is not allowed, because
# it holds to much potential to set the whole
# column unintentionally.
with pytest.raises(ValueError):
- flags[c] = 888.
+ flags[c] = 888.0
vector = var.copy()
- vector[:] = 333.
+ vector[:] = 333.0
flags[mask, c] = vector
- assert all(flags[c].loc[mask] == 333.)
- assert all(flags[c].loc[~mask] != 333.)
+ assert all(flags[c].loc[mask] == 333.0)
+ assert all(flags[c].loc[~mask] != 333.0)
# works with any that pandas eat, eg with numpy
- vector[:] = 444.
+ vector[:] = 444.0
vector = vector.to_numpy()
flags[mask, c] = vector
- assert all(flags[c].loc[mask] == 444.)
- assert all(flags[c].loc[~mask] != 444.)
+ assert all(flags[c].loc[mask] == 444.0)
+ assert all(flags[c].loc[~mask] != 444.0)
# test length miss-match (mask)
if len(mask) > 1:
@@ -196,8 +201,10 @@ def test_set_flags_with_mask(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[s
flags[mask, c] = wrong_len
-@pytest.mark.parametrize('data', data)
-def test_set_flags_with_index(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
+@pytest.mark.parametrize("data", data)
+def test_set_flags_with_index(
+ data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]
+):
flags = Flags(data)
for c in flags.columns:
@@ -205,23 +212,23 @@ def test_set_flags_with_index(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[
mask = var == UNFLAGGED
index = mask[mask].index
- scalar = 222.
+ scalar = 222.0
flags[index, c] = scalar
- assert all(flags[c].loc[mask] == 222.)
- assert all(flags[c].loc[~mask] != 222.)
+ assert all(flags[c].loc[mask] == 222.0)
+ assert all(flags[c].loc[~mask] != 222.0)
vector = var.copy()
- vector[:] = 333.
+ vector[:] = 333.0
flags[index, c] = vector
- assert all(flags[c].loc[mask] == 333.)
- assert all(flags[c].loc[~mask] != 333.)
+ assert all(flags[c].loc[mask] == 333.0)
+ assert all(flags[c].loc[~mask] != 333.0)
# works with any that pandas eat, eg with numpy
- vector[:] = 444.
+ vector[:] = 444.0
vector = vector.to_numpy()
flags[index, c] = vector
- assert all(flags[c].loc[mask] == 444.)
- assert all(flags[c].loc[~mask] != 444.)
+ assert all(flags[c].loc[mask] == 444.0)
+ assert all(flags[c].loc[~mask] != 444.0)
# test length miss-match (value)
if len(vector) > 1:
@@ -231,24 +238,26 @@ def test_set_flags_with_index(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[
def test_cache():
- arr = np.array([
- [0, 0, 0, 0],
- [0, 1, 2, 3],
- [0, 1, 2, 3],
- ])
- data = pd.DataFrame(arr, dtype=float, columns=list('abcd'))
+ arr = np.array(
+ [
+ [0, 0, 0, 0],
+ [0, 1, 2, 3],
+ [0, 1, 2, 3],
+ ]
+ )
+ data = pd.DataFrame(arr, dtype=float, columns=list("abcd"))
flags = Flags(data)
# cache empty
assert flags._cache == {}
# invoke caching
- flags['a']
- assert 'a' in flags._cache
+ flags["a"]
+ assert "a" in flags._cache
# clears cache
- flags['a'] = pd.Series([0, 0, 0], dtype=float)
- assert 'a' not in flags._cache
+ flags["a"] = pd.Series([0, 0, 0], dtype=float)
+ assert "a" not in flags._cache
# cache all
flags.toDios()
@@ -256,7 +265,7 @@ def test_cache():
assert c in flags._cache
# cache survive renaming
- flags.columns = list('xyzq')
+ flags.columns = list("xyzq")
for c in flags.columns:
assert c in flags._cache
@@ -269,7 +278,7 @@ def _validate_flags_equals_frame(flags, df):
assert df[c].equals(flags[c]) # respects nan's
-@pytest.mark.parametrize('data', data)
+@pytest.mark.parametrize("data", data)
def test_to_dios(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
flags = Flags(data)
df = flags.toDios()
@@ -278,7 +287,7 @@ def test_to_dios(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Serie
_validate_flags_equals_frame(flags, df)
-@pytest.mark.parametrize('data', data)
+@pytest.mark.parametrize("data", data)
def test_to_frame(data: Union[pd.DataFrame, dios.DictOfSeries, Dict[str, pd.Series]]):
flags = Flags(data)
df = flags.toFrame()
diff --git a/tests/core/test_history.py b/tests/core/test_history.py
index 3b3f87a58..ebeab59ba 100644
--- a/tests/core/test_history.py
+++ b/tests/core/test_history.py
@@ -9,69 +9,72 @@ from saqc.core.history import History
# see #GH143 combined backtrack
# (adjusted to current implementation)
example1 = (
-
# flags
- np.array([
- [0, np.nan, 50, 99, np.nan],
- [0, np.nan, 50, np.nan, 25],
- [0, 99, 99, 99, 25],
- [0, 99, np.nan, np.nan, 25],
- ]),
-
+ np.array(
+ [
+ [0, np.nan, 50, 99, np.nan],
+ [0, np.nan, 50, np.nan, 25],
+ [0, 99, 99, 99, 25],
+ [0, 99, np.nan, np.nan, 25],
+ ]
+ ),
# mask
- np.array([
- [0, 0, 0, 1],
- [1, 1, 1, 1],
- [1, 1, 1, 1],
- [1, 1, 1, 1],
- ]),
-
+ np.array(
+ [
+ [0, 0, 0, 1],
+ [1, 1, 1, 1],
+ [1, 1, 1, 1],
+ [1, 1, 1, 1],
+ ]
+ ),
# expected from max()
- np.array([99, 25, 25, 25])
+ np.array([99, 25, 25, 25]),
)
# see #GH143
example2 = (
-
# flags
- np.array([
- [0, 99, np.nan, 0],
- [0, np.nan, 99, np.nan],
- [0, np.nan, np.nan, np.nan],
- [0, np.nan, np.nan, 0],
- ]),
-
+ np.array(
+ [
+ [0, 99, np.nan, 0],
+ [0, np.nan, 99, np.nan],
+ [0, np.nan, np.nan, np.nan],
+ [0, np.nan, np.nan, 0],
+ ]
+ ),
# mask
- np.array([
- [0, 0, 0, 1],
- [1, 1, 1, 1],
- [1, 1, 1, 1],
- [0, 0, 0, 1],
- ]),
-
+ np.array(
+ [
+ [0, 0, 0, 1],
+ [1, 1, 1, 1],
+ [1, 1, 1, 1],
+ [0, 0, 0, 1],
+ ]
+ ),
# expected from max()
- np.array([0, 99, 0, 0])
+ np.array([0, 99, 0, 0]),
)
data = [
-
np.array([[]]),
np.zeros((1, 1)),
np.zeros((3, 4)),
np.ones((3, 4)),
np.ones((3, 4)) * np.nan,
-
- np.array([
- [0, 0, 0, 0],
- [0, 1, 2, 3],
- [0, 1, 2, 3],
- ]),
-
- np.array([
- [0, 0, 0, 0],
- [0, 1, np.nan, 3],
- [0, 1, 2, 3],
- ]),
+ np.array(
+ [
+ [0, 0, 0, 0],
+ [0, 1, 2, 3],
+ [0, 1, 2, 3],
+ ]
+ ),
+ np.array(
+ [
+ [0, 0, 0, 0],
+ [0, 1, np.nan, 3],
+ [0, 1, 2, 3],
+ ]
+ ),
]
@@ -119,7 +122,7 @@ def is_equal(hist1: History, hist2: History):
return hist1.hist.equals(hist2.hist) and hist1.mask.equals(hist2.mask)
-@pytest.mark.parametrize('data', data + [None])
+@pytest.mark.parametrize("data", data + [None])
def test_init(data: np.array):
# init
df = pd.DataFrame(data, dtype=float)
@@ -140,7 +143,7 @@ def test_init(data: np.array):
assert is_equal(hist, fast)
-@pytest.mark.parametrize('data', data + [None])
+@pytest.mark.parametrize("data", data + [None])
def test_init_with_mask(data: np.array):
# init
df = pd.DataFrame(data, dtype=float)
@@ -163,7 +166,7 @@ def test_init_with_mask(data: np.array):
assert is_equal(hist, fast)
-@pytest.mark.parametrize('data', data + [None])
+@pytest.mark.parametrize("data", data + [None])
def test_copy(data):
# init
df = pd.DataFrame(data, dtype=float)
@@ -187,7 +190,7 @@ def test_copy(data):
assert shallow.mask is hist.mask
-@pytest.mark.parametrize('data', data + [None])
+@pytest.mark.parametrize("data", data + [None])
def test_reindex_trivial_cases(data):
df = pd.DataFrame(data, dtype=float)
orig = History(hist=df)
@@ -200,7 +203,7 @@ def test_reindex_trivial_cases(data):
check_invariants(hist)
-@pytest.mark.parametrize('data', data + [None])
+@pytest.mark.parametrize("data", data + [None])
def test_reindex_missing_indicees(data):
df = pd.DataFrame(data, dtype=float)
hist = History(hist=df)
@@ -211,7 +214,7 @@ def test_reindex_missing_indicees(data):
check_invariants(hist)
-@pytest.mark.parametrize('data', data + [None])
+@pytest.mark.parametrize("data", data + [None])
def test_reindex_extra_indicees(data):
df = pd.DataFrame(data, dtype=float)
hist = History(hist=df)
@@ -222,7 +225,7 @@ def test_reindex_extra_indicees(data):
check_invariants(hist)
-@pytest.fixture(scope='module')
+@pytest.fixture(scope="module")
def __hist():
# this FH is filled by
# - test_append
@@ -230,13 +233,16 @@ def __hist():
return History()
-@pytest.mark.parametrize('s, max_val', [
- (pd.Series(0, index=range(6), dtype=float), 0),
- (pd.Series(1, index=range(6), dtype=float), 1),
- (pd.Series(np.nan, index=range(6), dtype=float), 1),
- (pd.Series(1, index=range(6), dtype=float), 1),
- (pd.Series(0, index=range(6), dtype=float), 1),
-])
+@pytest.mark.parametrize(
+ "s, max_val",
+ [
+ (pd.Series(0, index=range(6), dtype=float), 0),
+ (pd.Series(1, index=range(6), dtype=float), 1),
+ (pd.Series(np.nan, index=range(6), dtype=float), 1),
+ (pd.Series(1, index=range(6), dtype=float), 1),
+ (pd.Series(0, index=range(6), dtype=float), 1),
+ ],
+)
def test_append(__hist, s, max_val):
hist = __hist
hist.append(s, force=False)
@@ -246,12 +252,15 @@ def test_append(__hist, s, max_val):
# this test append more rows to the resulting
# FH from the former test
-@pytest.mark.parametrize('s, max_val', [
- (pd.Series(0, index=range(6), dtype=float), 0),
- (pd.Series(1, index=range(6), dtype=float), 1),
- (pd.Series(np.nan, index=range(6), dtype=float), 1),
- (pd.Series(0, index=range(6), dtype=float), 0),
-])
+@pytest.mark.parametrize(
+ "s, max_val",
+ [
+ (pd.Series(0, index=range(6), dtype=float), 0),
+ (pd.Series(1, index=range(6), dtype=float), 1),
+ (pd.Series(np.nan, index=range(6), dtype=float), 1),
+ (pd.Series(0, index=range(6), dtype=float), 0),
+ ],
+)
def test_append_force(__hist, s, max_val):
hist = __hist
hist.append(s, force=True)
diff --git a/tests/core/test_reader.py b/tests/core/test_reader.py
index dd1941cf8..3efec94fe 100644
--- a/tests/core/test_reader.py
+++ b/tests/core/test_reader.py
@@ -26,7 +26,11 @@ def test_packagedConfig():
config_path = path / "config_ci.csv"
data_path = path / "data.csv"
- data = pd.read_csv(data_path, index_col=0, parse_dates=True,)
+ data = pd.read_csv(
+ data_path,
+ index_col=0,
+ parse_dates=True,
+ )
saqc = SaQC(dios.DictOfSeries(data)).readConfig(config_path)
saqc.getResult()
@@ -45,7 +49,9 @@ def test_variableRegex(data):
for regex, expected in tests:
fobj = writeIO(header + "\n" + f"{regex} ; flagtools.flagDummy()")
saqc = SaQC(data).readConfig(fobj)
- expansion = saqc._expandFields(saqc._planned[0][0], saqc._planned[0][2], data.columns)
+ expansion = saqc._expandFields(
+ saqc._planned[0][0], saqc._planned[0][2], data.columns
+ )
result = [s.field for s, _ in expansion]
assert np.all(result == expected)
@@ -130,7 +136,7 @@ def test_supportedArguments(data):
# TODO: necessary?
- @register(masking='field')
+ @register(masking="field")
def func(data, field, flags, kwarg, **kwargs):
return data, flags
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index 6fddf9a6d..400b96a8e 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -22,7 +22,11 @@ from tests.common import initData
def _genTranslators():
for dtype in (str, float, int):
- flags = {dtype(-2): UNFLAGGED, dtype(-1): BAD, **{dtype(f*10): float(f) for f in range(10)}}
+ flags = {
+ dtype(-2): UNFLAGGED,
+ dtype(-1): BAD,
+ **{dtype(f * 10): float(f) for f in range(10)},
+ }
translator = Translator(flags)
yield flags, translator
@@ -32,7 +36,9 @@ def _genFlags(data: Dict[str, Union[Sequence, pd.Series]]) -> Flags:
flags = DictOfSeries()
for k, v in data.items():
if not isinstance(v, pd.Series):
- v = pd.Series(v, index=pd.date_range("2012-01-01", freq="1D", periods=len(v)))
+ v = pd.Series(
+ v, index=pd.date_range("2012-01-01", freq="1D", periods=len(v))
+ )
flags[k] = v
return Flags(flags)
@@ -69,7 +75,6 @@ def test_backwardTranslationFail():
translator.backward(flags, None)
-
def test_dmpTranslator():
Selector = namedtuple("Selector", ["field"])
@@ -87,13 +92,21 @@ def test_dmpTranslator():
(Selector("var2"), Function("flagFoo")),
]
tflags = translator.backward(flags, to_call)
- assert set(tflags.columns.get_level_values(1)) == {"quality_flag", "quality_comment", "quality_cause"}
+ assert set(tflags.columns.get_level_values(1)) == {
+ "quality_flag",
+ "quality_comment",
+ "quality_cause",
+ }
assert (tflags.loc[:, ("var1", "quality_flag")] == "DOUBTFUL").all(axis=None)
- assert (tflags.loc[:, ("var1", "quality_comment")] == '{"test": "flagBar"}').all(axis=None)
+ assert (tflags.loc[:, ("var1", "quality_comment")] == '{"test": "flagBar"}').all(
+ axis=None
+ )
assert (tflags.loc[:, ("var2", "quality_flag")] == "BAD").all(axis=None)
- assert (tflags.loc[:, ("var2", "quality_comment")] == '{"test": "flagFoo"}').all(axis=None)
+ assert (tflags.loc[:, ("var2", "quality_comment")] == '{"test": "flagFoo"}').all(
+ axis=None
+ )
assert (tflags.loc[:, ("var3", "quality_comment")] == '{"test": ""}').all(axis=None)
@@ -118,9 +131,7 @@ def test_positionalTranslatorIntegration():
translator = PositionalTranslator()
saqc = SaQC(data=data, translator=translator)
- saqc = (saqc
- .breaks.flagMissing(col)
- .outliers.flagRange(col, min=3, max=10))
+ saqc = saqc.breaks.flagMissing(col).outliers.flagRange(col, min=3, max=10)
data, flags = saqc.getResult()
for field in flags.columns:
@@ -139,13 +150,13 @@ def test_dmpTranslatorIntegration():
translator = DmpTranslator()
saqc = SaQC(data=data, translator=translator)
- saqc = (saqc
- .breaks.flagMissing(col)
- .outliers.flagRange(col, min=3, max=10))
+ saqc = saqc.breaks.flagMissing(col).outliers.flagRange(col, min=3, max=10)
data, flags = saqc.getResult()
qflags = flags.xs("quality_flag", axis="columns", level=1)
- qfunc = flags.xs("quality_comment", axis="columns", level=1).applymap(lambda v: json.loads(v)["test"])
+ qfunc = flags.xs("quality_comment", axis="columns", level=1).applymap(
+ lambda v: json.loads(v)["test"]
+ )
qcause = flags.xs("quality_cause", axis="columns", level=1)
assert qflags.isin(translator._forward.keys()).all(axis=None)
@@ -154,10 +165,11 @@ def test_dmpTranslatorIntegration():
round_trip = translator.backward(translator.forward(flags), saqc._computed)
assert round_trip.xs("quality_flag", axis="columns", level=1).equals(qflags)
- assert (round_trip
- .xs("quality_comment", axis="columns", level=1)
- .applymap(lambda v: json.loads(v)["test"] == "")
- .all(axis=None))
+ assert (
+ round_trip.xs("quality_comment", axis="columns", level=1)
+ .applymap(lambda v: json.loads(v)["test"] == "")
+ .all(axis=None)
+ )
def _buildupSaQCObjects():
@@ -173,9 +185,9 @@ def _buildupSaQCObjects():
out = []
for _ in range(2):
saqc = SaQC(data=data, flags=flags)
- saqc = (saqc
- .breaks.flagMissing(col, to_mask=False)
- .outliers.flagRange(col, min=3, max=10, to_mask=False))
+ saqc = saqc.breaks.flagMissing(col, to_mask=False).outliers.flagRange(
+ col, min=3, max=10, to_mask=False
+ )
saqc = saqc.evaluate()
flags = saqc._flags
out.append(saqc)
@@ -212,6 +224,7 @@ def test_positionalTranslationPreservesFlags():
got = tflags2[k].str.slice(start=1)
assert expected.equals(got)
+
def test_dmpTranslationPreservesFlags():
saqc1, saqc2 = _buildupSaQCObjects()
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 8449ef6fa..a14d60f75 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -30,23 +30,26 @@ def course_1(char_dict):
the resulting drop/raise per value equals: (peak_level - initial_level) / (0.5*(periods-2))
periods number better be even!
"""
+
def fix_funk(
- freq="10min",
- periods=10,
- initial_level=0,
- peak_level=10,
- initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
- char_dict=char_dict,
- name='data'
+ freq="10min",
+ periods=10,
+ initial_level=0,
+ peak_level=10,
+ initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
+ char_dict=char_dict,
+ name="data",
):
t_index = pd.date_range(initial_index, freq=freq, periods=periods)
left = np.linspace(initial_level, peak_level, int(np.floor(len(t_index) / 2)))
right = np.linspace(peak_level, initial_level, int(np.ceil(len(t_index) / 2)))
s = pd.Series(np.append(left, right), index=t_index)
- char_dict["raise"] = s.index[1: int(np.floor(len(t_index) / 2))]
- char_dict["drop"] = s.index[int(np.floor(len(t_index) / 2) + 1):]
- char_dict["peak"] = s.index[int(np.floor(len(t_index) / 2)) - 1: int(np.floor(len(t_index) / 2)) + 1]
+ char_dict["raise"] = s.index[1 : int(np.floor(len(t_index) / 2))]
+ char_dict["drop"] = s.index[int(np.floor(len(t_index) / 2) + 1) :]
+ char_dict["peak"] = s.index[
+ int(np.floor(len(t_index) / 2)) - 1 : int(np.floor(len(t_index) / 2)) + 1
+ ]
data = DictOfSeries(data=s, columns=[name])
return data, char_dict
@@ -65,13 +68,13 @@ def course_2(char_dict):
"""
# SINGLE_SPIKE
def fix_funk(
- freq="10min",
- periods=10,
- initial_level=0,
- final_level=2,
- out_val=5,
- initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
- char_dict=char_dict,
+ freq="10min",
+ periods=10,
+ initial_level=0,
+ final_level=2,
+ out_val=5,
+ initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
+ char_dict=char_dict,
):
t_index = pd.date_range(initial_index, freq=freq, periods=periods)
data = np.linspace(initial_level, final_level, int(np.floor(len(t_index))))
@@ -100,15 +103,20 @@ def course_test(char_dict):
same as test pattern for first three values, than constant function
"""
- def fix_funk(freq='1 D',
- initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0), out_val=5, char_dict=char_dict):
+
+ def fix_funk(
+ freq="1 D",
+ initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
+ out_val=5,
+ char_dict=char_dict,
+ ):
t_index = pd.date_range(initial_index, freq=freq, periods=100)
data = pd.Series(data=0, index=t_index)
data.iloc[2] = out_val
data.iloc[3] = out_val
- data = DictOfSeries(data=data, columns=['data'])
+ data = DictOfSeries(data=data, columns=["data"])
return data, char_dict
return fix_funk
@@ -126,16 +134,17 @@ def course_3(char_dict):
number of periods better be even!
chrowd_size * crowd_spacing better be less then freq[minutes].
"""
+
def fix_funk(
- freq="10min",
- periods=10,
- initial_level=0,
- final_level=2,
- out_val=-5,
- initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
- char_dict=char_dict,
- crowd_size=5,
- crowd_spacing=1,
+ freq="10min",
+ periods=10,
+ initial_level=0,
+ final_level=2,
+ out_val=-5,
+ initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
+ char_dict=char_dict,
+ crowd_size=5,
+ crowd_spacing=1,
):
t_index = pd.date_range(initial_index, freq=freq, periods=periods)
@@ -143,13 +152,18 @@ def course_3(char_dict):
data = pd.Series(data=data, index=t_index)
ind1 = data.index[int(np.floor(periods / 2))]
- dates = [ind1 + crowd_spacing * pd.Timedelta(f"{k}min") for k in range(1, crowd_size + 1)]
+ dates = [
+ ind1 + crowd_spacing * pd.Timedelta(f"{k}min")
+ for k in range(1, crowd_size + 1)
+ ]
insertion_index = pd.DatetimeIndex(dates)
data.iloc[int(np.floor(periods / 2))] = out_val
data = data.append(pd.Series(data=out_val, index=insertion_index))
data.sort_index(inplace=True)
- anomaly_index = insertion_index.insert(0, data.index[int(np.floor(periods / 2))])
+ anomaly_index = insertion_index.insert(
+ 0, data.index[int(np.floor(periods / 2))]
+ )
if out_val > data.iloc[int(np.floor(periods / 2) - 1)]:
kind = "raise"
@@ -176,18 +190,18 @@ def course_4(char_dict):
"""
def fix_funk(
- freq="10min",
- periods=10,
- base_level=0,
- out_val=5,
- initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
- char_dict=char_dict,
+ freq="10min",
+ periods=10,
+ base_level=0,
+ out_val=5,
+ initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
+ char_dict=char_dict,
):
t_index = pd.date_range(initial_index, freq=freq, periods=periods)
data = pd.Series(data=base_level, index=t_index)
- data[int(len(t_index) / 2):: 2] = out_val
- char_dict["raise"] = t_index[int(len(t_index) / 2):: 2]
- char_dict["return"] = t_index[int((len(t_index) / 2) + 1):: 2]
+ data[int(len(t_index) / 2) :: 2] = out_val
+ char_dict["raise"] = t_index[int(len(t_index) / 2) :: 2]
+ char_dict["return"] = t_index[int((len(t_index) / 2) + 1) :: 2]
data = DictOfSeries(data=data, columns=["data"])
return data, char_dict
@@ -207,13 +221,13 @@ def course_5(char_dict):
"""
def fix_funk(
- freq="10min",
- periods=10,
- nan_slice=slice(0, None, 5),
- initial_level=0,
- final_level=10,
- initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
- char_dict=char_dict,
+ freq="10min",
+ periods=10,
+ nan_slice=slice(0, None, 5),
+ initial_level=0,
+ final_level=10,
+ initial_index=pd.Timestamp(2000, 1, 1, 0, 0, 0),
+ char_dict=char_dict,
):
t_index = pd.date_range(initial_index, freq=freq, periods=periods)
values = np.linspace(initial_level, final_level, periods)
diff --git a/tests/funcs/test_constants_detection.py b/tests/funcs/test_constants_detection.py
index a7a7b5b82..48c95744c 100644
--- a/tests/funcs/test_constants_detection.py
+++ b/tests/funcs/test_constants_detection.py
@@ -13,7 +13,9 @@ from tests.common import initData
@pytest.fixture
def data():
- constants_data = initData(1, start_date="2011-01-01 00:00:00", end_date="2011-01-01 03:00:00", freq="5min")
+ constants_data = initData(
+ 1, start_date="2011-01-01 00:00:00", end_date="2011-01-01 03:00:00", freq="5min"
+ )
constants_data.iloc[5:25] = 200
return constants_data
@@ -22,7 +24,9 @@ def test_constants_flagBasic(data):
expected = np.arange(5, 22)
field, *_ = data.columns
flags = initFlagsLike(data)
- data, flags_result = flagConstants(data, field, flags, window="15Min", thresh=0.1, flag=BAD)
+ data, flags_result = flagConstants(
+ data, field, flags, window="15Min", thresh=0.1, flag=BAD
+ )
flagscol = flags_result[field]
assert np.all(flagscol[expected] == BAD)
diff --git a/tests/funcs/test_functions.py b/tests/funcs/test_functions.py
index 7d625d71e..9c8f7e4b7 100644
--- a/tests/funcs/test_functions.py
+++ b/tests/funcs/test_functions.py
@@ -5,7 +5,11 @@ import dios
from saqc.constants import *
from saqc.core import initFlagsLike
-from saqc.funcs.drift import flagDriftFromNorm, flagDriftFromReference, flagDriftFromScaledNorm
+from saqc.funcs.drift import (
+ flagDriftFromNorm,
+ flagDriftFromReference,
+ flagDriftFromScaledNorm,
+)
from saqc.funcs.outliers import flagCrossStatistic, flagRange
from saqc.funcs.flagtools import flagManual, forceFlags, clearFlags
from saqc.funcs.tools import drop, copy, mask
@@ -41,9 +45,28 @@ def test_flagSesonalRange(data, field):
nyears = len(data[field].index.year.unique())
tests = [
- ({"min": 1, "max": 100, "startmonth": 7, "startday": 1, "endmonth": 8, "endday": 31, }, 31 * 2 * nyears // 2,),
(
- {"min": 1, "max": 100, "startmonth": 12, "startday": 16, "endmonth": 1, "endday": 15, }, 31 * nyears // 2 + 1,),
+ {
+ "min": 1,
+ "max": 100,
+ "startmonth": 7,
+ "startday": 1,
+ "endmonth": 8,
+ "endday": 31,
+ },
+ 31 * 2 * nyears // 2,
+ ),
+ (
+ {
+ "min": 1,
+ "max": 100,
+ "startmonth": 12,
+ "startday": 16,
+ "endmonth": 1,
+ "endday": 15,
+ },
+ 31 * nyears // 2 + 1,
+ ),
]
for test, expected in tests:
@@ -54,11 +77,21 @@ def test_flagSesonalRange(data, field):
data, flags = copy(data, field, flags, field + "_masked")
data, flags = mask(
- data, newfield, flags,
- mode='periodic', period_start=start, period_end=end, include_bounds=True, flag=BAD
+ data,
+ newfield,
+ flags,
+ mode="periodic",
+ period_start=start,
+ period_end=end,
+ include_bounds=True,
+ flag=BAD,
+ )
+ data, flags = flagRange(
+ data, newfield, flags, min=test["min"], max=test["max"], flag=BAD
+ )
+ data, flags = reindexFlags(
+ data, field, flags, method="match", source=newfield, flag=BAD
)
- data, flags = flagRange(data, newfield, flags, min=test['min'], max=test['max'], flag=BAD)
- data, flags = reindexFlags(data, field, flags, method='match', source=newfield, flag=BAD)
data, flags = drop(data, newfield, flags)
flagged = flags[field] > UNFLAGGED
assert flagged.sum() == expected
@@ -102,13 +135,20 @@ def test_flagIsolated(data, field):
# 2016-01-08 7.0 -inf
# .. .. ..
- _, flags_result = flagIsolated(data, field, flags, group_window="1D", gap_window="2.1D", flag=BAD)
+ _, flags_result = flagIsolated(
+ data, field, flags, group_window="1D", gap_window="2.1D", flag=BAD
+ )
assert flags_result[field].iloc[[3, 5]].all()
data, flags_result = flagIsolated(
- data, field, flags_result,
- group_window="2D", gap_window="2.1D", continuation_range="1.1D", flag=BAD
+ data,
+ field,
+ flags_result,
+ group_window="2D",
+ gap_window="2.1D",
+ continuation_range="1.1D",
+ flag=BAD,
)
assert flags_result[field].iloc[[3, 5, 13, 14]].all()
@@ -124,7 +164,9 @@ def test_flagCrossScoring(dat):
s2 = pd.Series(data=s2.values, index=s1.index)
data = dios.DictOfSeries([s1, s2], columns=["data1", "data2"])
flags = initFlagsLike(data)
- _, flags_result = flagCrossStatistic(data, field, flags, fields=fields, thresh=3, cross_stat=np.mean, flag=BAD)
+ _, flags_result = flagCrossStatistic(
+ data, field, flags, fields=fields, thresh=3, cross_stat=np.mean, flag=BAD
+ )
for field in fields:
isflagged = flags_result[field] > UNFLAGGED
assert isflagged[characteristics["raise"]].all()
@@ -154,7 +196,9 @@ def test_flagManual(data, field):
assert isflagged[isflagged].index.equals(index_exp)
# flag not exist in mdata
- _, fl = flagManual(*args, mdata=mdata, mflag="i do not exist", method="ontime", flag=BAD)
+ _, fl = flagManual(
+ *args, mdata=mdata, mflag="i do not exist", method="ontime", flag=BAD
+ )
isflagged = fl[field] > UNFLAGGED
assert isflagged[isflagged].index.equals(pd.DatetimeIndex([]))
@@ -214,37 +258,44 @@ def test_flagManual(data, field):
@pytest.mark.parametrize("dat", [pytest.lazy_fixture("course_1")])
def test_flagDriftFromNormal(dat):
- data = dat(periods=200, peak_level=5, name='d1')[0]
- data['d2'] = dat(periods=200, peak_level=10, name='d2')[0]['d2']
- data['d3'] = dat(periods=200, peak_level=100, name='d3')[0]['d3']
- data['d4'] = 3 + 4 * data['d1']
- data['d5'] = 3 + 4 * data['d1']
+ data = dat(periods=200, peak_level=5, name="d1")[0]
+ data["d2"] = dat(periods=200, peak_level=10, name="d2")[0]["d2"]
+ data["d3"] = dat(periods=200, peak_level=100, name="d3")[0]["d3"]
+ data["d4"] = 3 + 4 * data["d1"]
+ data["d5"] = 3 + 4 * data["d1"]
flags = initFlagsLike(data)
data_norm, flags_norm = flagDriftFromNorm(
- data, 'dummy', flags,
- ['d1', 'd2', 'd3'],
+ data,
+ "dummy",
+ flags,
+ ["d1", "d2", "d3"],
segment_freq="200min",
norm_spread=5,
flag=BAD,
)
data_ref, flags_ref = flagDriftFromReference(
- data, 'd1', flags,
- ['d1', 'd2', 'd3'],
+ data,
+ "d1",
+ flags,
+ ["d1", "d2", "d3"],
segment_freq="3D",
thresh=20,
flag=BAD,
)
data_scale, flags_scale = flagDriftFromScaledNorm(
- data, 'dummy', flags,
- ['d1', 'd3'], ['d4', 'd5'],
+ data,
+ "dummy",
+ flags,
+ ["d1", "d3"],
+ ["d4", "d5"],
segment_freq="3D",
thresh=20,
norm_spread=5,
flag=BAD,
)
- assert all(flags_norm['d3'] > UNFLAGGED)
- assert all(flags_ref['d3'] > UNFLAGGED)
- assert all(flags_scale['d3'] > UNFLAGGED)
+ assert all(flags_norm["d3"] > UNFLAGGED)
+ assert all(flags_ref["d3"] > UNFLAGGED)
+ assert all(flags_scale["d3"] > UNFLAGGED)
diff --git a/tests/funcs/test_generic_api_functions.py b/tests/funcs/test_generic_api_functions.py
index 64b922fae..710fbecea 100644
--- a/tests/funcs/test_generic_api_functions.py
+++ b/tests/funcs/test_generic_api_functions.py
@@ -12,7 +12,7 @@ from saqc import SaQC
from tests.common import initData, flagAll
-register(masking='field')(flagAll)
+register(masking="field")(flagAll)
@pytest.fixture
@@ -32,12 +32,14 @@ def test_addFieldProcGeneric(data):
saqc = SaQC(data=data)
func = lambda: pd.Series([])
- data, flags = saqc.generic.process("tmp1", func, flag=BAD ).getResult(raw=True)
+ data, flags = saqc.generic.process("tmp1", func, flag=BAD).getResult(raw=True)
assert "tmp1" in data.columns and data["tmp1"].empty
func = lambda var1, var2: var1 + var2
data, flags = saqc.generic.process("tmp2", func, flag=BAD).getResult()
- assert "tmp2" in data.columns and (data["tmp2"] == data["var1"] + data["var2"]).all(axis=None)
+ assert "tmp2" in data.columns and (data["tmp2"] == data["var1"] + data["var2"]).all(
+ axis=None
+ )
def test_mask(data):
@@ -45,9 +47,17 @@ def test_mask(data):
data_org = data.copy(deep=True)
mean = data["var1"] / 2
- data, _ = saqc.generic.process("var1", lambda var1: mask(var1 < mean), flag=BAD).getResult()
- assert ((data["var1"].isna()) == (data_org["var1"] < 10) & data_org["var1"].isna()).all(axis=None)
+ data, _ = saqc.generic.process(
+ "var1", lambda var1: mask(var1 < mean), flag=BAD
+ ).getResult()
+ assert (
+ (data["var1"].isna()) == (data_org["var1"] < 10) & data_org["var1"].isna()
+ ).all(axis=None)
- data, flags = saqc.generic.process("tmp", lambda var1: mask(var1 < mean), flag=BAD).getResult()
+ data, flags = saqc.generic.process(
+ "tmp", lambda var1: mask(var1 < mean), flag=BAD
+ ).getResult()
assert ("tmp" in data.columns) and ("tmp" in flags.columns)
- assert ((data["tmp"].isna()) == (data_org["var1"] < 10) & data_org["var1"].isna()).all(axis=None)
+ assert (
+ (data["tmp"].isna()) == (data_org["var1"] < 10) & data_org["var1"].isna()
+ ).all(axis=None)
diff --git a/tests/funcs/test_generic_config_functions.py b/tests/funcs/test_generic_config_functions.py
index a47407866..ea3d8cf5e 100644
--- a/tests/funcs/test_generic_config_functions.py
+++ b/tests/funcs/test_generic_config_functions.py
@@ -30,7 +30,12 @@ def data_diff():
col1 = data[data.columns[1]]
mid = len(col0) // 2
offset = len(col0) // 8
- return dios.DictOfSeries(data={col0.name: col0.iloc[: mid + offset], col1.name: col1.iloc[mid - offset :],})
+ return dios.DictOfSeries(
+ data={
+ col0.name: col0.iloc[: mid + offset],
+ col1.name: col1.iloc[mid - offset :],
+ }
+ )
def _compileGeneric(expr, flags):
@@ -133,7 +138,10 @@ def test_nonReduncingBuiltins(data):
(f"abs({this})", np.abs(data[this])),
(f"log({this})", np.log(data[this])),
(f"exp({this})", np.exp(data[this])),
- (f"ismissing(mask({this} < {mean}))", data[this].mask(data[this] < mean).isna()),
+ (
+ f"ismissing(mask({this} < {mean}))",
+ data[this].mask(data[this] < mean).isna(),
+ ),
]
for test, expected in tests:
@@ -213,7 +221,10 @@ def test_isflagged(data):
(f"isflagged({var1}, flag=BAD)", flags[var1] >= BAD),
(f"isflagged({var1}, UNFLAGGED, '==')", flags[var1] == UNFLAGGED),
(f"~isflagged({var2})", flags[var2] == UNFLAGGED),
- (f"~({var2}>999) & (~isflagged({var2}))", ~(data[var2] > 999) & (flags[var2] == UNFLAGGED)),
+ (
+ f"~({var2}>999) & (~isflagged({var2}))",
+ ~(data[var2] > 999) & (flags[var2] == UNFLAGGED),
+ ),
]
for i, (test, expected) in enumerate(tests):
@@ -226,7 +237,7 @@ def test_isflagged(data):
raise
# test bad combination
- for comp in ['>', '>=', '==', '!=', '<', '<=']:
+ for comp in [">", ">=", "==", "!=", "<", "<="]:
fails = f"isflagged({var1}, comparator='{comp}')"
func = _compileGeneric(f"generic.flag(func={fails}, flag=BAD)", flags)
@@ -274,7 +285,7 @@ def test_callableArgumentsUnary(data):
window = 5
- @register(masking='field')
+ @register(masking="field")
def testFuncUnary(data, field, flags, func, **kwargs):
data[field] = data[field].rolling(window=window).apply(func)
return data, initFlagsLike(data)
@@ -303,7 +314,7 @@ def test_callableArgumentsUnary(data):
def test_callableArgumentsBinary(data):
var1, var2 = data.columns[:2]
- @register(masking='field')
+ @register(masking="field")
def testFuncBinary(data, field, flags, func, **kwargs):
data[field] = func(data[var1], data[var2])
return data, initFlagsLike(data)
diff --git a/tests/funcs/test_harm_funcs.py b/tests/funcs/test_harm_funcs.py
index a8606ca72..4bc0db242 100644
--- a/tests/funcs/test_harm_funcs.py
+++ b/tests/funcs/test_harm_funcs.py
@@ -21,7 +21,9 @@ from tests.common import checkDataFlagsInvariants
@pytest.fixture
def data():
- index = pd.date_range(start="1.1.2011 00:00:00", end="1.1.2011 01:00:00", freq="15min")
+ index = pd.date_range(
+ start="1.1.2011 00:00:00", end="1.1.2011 01:00:00", freq="15min"
+ )
index = index.insert(2, pd.Timestamp(2011, 1, 1, 0, 29, 0))
index = index.insert(2, pd.Timestamp(2011, 1, 1, 0, 28, 0))
index = index.insert(5, pd.Timestamp(2011, 1, 1, 0, 32, 0))
@@ -35,18 +37,22 @@ def data():
return data
-@pytest.mark.parametrize('func, kws', [
- ('linear', dict()),
- ('shift', dict(method="nshift")),
- ('interpolate', dict(method="spline")),
- ('aggregate', dict(value_func=np.nansum, method="nagg")),
-])
+@pytest.mark.parametrize(
+ "func, kws",
+ [
+ ("linear", dict()),
+ ("shift", dict(method="nshift")),
+ ("interpolate", dict(method="spline")),
+ ("aggregate", dict(value_func=np.nansum, method="nagg")),
+ ],
+)
def test_wrapper(data, func, kws):
- field = 'data'
+ field = "data"
freq = "15min"
flags = initFlagsLike(data)
import saqc
+
func = getattr(saqc.funcs, func)
data, flags = func(data, field, flags, freq, **kws)
@@ -58,18 +64,36 @@ def test_wrapper(data, func, kws):
@pytest.mark.parametrize("method", ["time", "polynomial"])
def test_gridInterpolation(data, method):
freq = "15min"
- field = 'data'
+ field = "data"
data = data[field]
data = (data * np.sin(data)).append(data.shift(1, "2h")).shift(1, "3s")
data = dios.DictOfSeries(data)
flags = initFlagsLike(data)
# we are just testing if the interpolation gets passed to the series without causing an error:
- res = interpolate(data, field, flags, freq, method=method, downcast_interpolation=True)
+ res = interpolate(
+ data, field, flags, freq, method=method, downcast_interpolation=True
+ )
if method == "polynomial":
- res = interpolate(data, field, flags, freq, order=2, method=method, downcast_interpolation=True)
- res = interpolate(data, field, flags, freq, order=10, method=method, downcast_interpolation=True)
+ res = interpolate(
+ data,
+ field,
+ flags,
+ freq,
+ order=2,
+ method=method,
+ downcast_interpolation=True,
+ )
+ res = interpolate(
+ data,
+ field,
+ flags,
+ freq,
+ order=10,
+ method=method,
+ downcast_interpolation=True,
+ )
# check minimal requirements
rdata, rflags = res
@@ -77,12 +101,15 @@ def test_gridInterpolation(data, method):
assert rdata[field].index.freq == pd.Timedelta(freq)
-@pytest.mark.parametrize('func, kws', [
- ('linear', dict()),
- ('shift', dict(method="nshift")),
- ('interpolate', dict(method="spline")),
- ('aggregate', dict(value_func=np.nansum, method="nagg")),
-])
+@pytest.mark.parametrize(
+ "func, kws",
+ [
+ ("linear", dict()),
+ ("shift", dict(method="nshift")),
+ ("interpolate", dict(method="spline")),
+ ("aggregate", dict(value_func=np.nansum, method="nagg")),
+ ],
+)
def test_flagsSurviveReshaping(func, kws):
"""
flagging -> reshaping -> test (flags also was reshaped correctly)
@@ -103,17 +130,19 @@ def test_flagsSurviveBackprojection():
pass
-@pytest.mark.parametrize("reshaper", ["nshift", "fshift", "bshift", "nagg", "bagg", "fagg", "interpolation"])
+@pytest.mark.parametrize(
+ "reshaper", ["nshift", "fshift", "bshift", "nagg", "bagg", "fagg", "interpolation"]
+)
def test_harmSingleVarIntermediateFlagging(data, reshaper):
flags = initFlagsLike(data)
- field = 'data'
+ field = "data"
pre_data = data.copy()
pre_flags = flags.copy()
data, flags = linear(data, field, flags, freq="15min")
checkDataFlagsInvariants(data, flags, field, identical=True)
- assert data[field].index.freq == pd.Timedelta('15min')
+ assert data[field].index.freq == pd.Timedelta("15min")
# flag something bad
flags[data[field].index[3:4], field] = BAD
@@ -123,7 +152,7 @@ def test_harmSingleVarIntermediateFlagging(data, reshaper):
assert data[field].equals(pre_data[field])
assert flags[field].index.equals(pre_flags[field].index)
- if 'agg' in reshaper:
+ if "agg" in reshaper:
if reshaper == "nagg":
start, end = 3, 7
elif reshaper == "fagg":
@@ -131,13 +160,13 @@ def test_harmSingleVarIntermediateFlagging(data, reshaper):
elif reshaper == "bagg":
start, end = 5, 7
else:
- raise NotImplementedError('untested test case')
+ raise NotImplementedError("untested test case")
assert all(flags[field].iloc[start:end] > UNFLAGGED)
assert all(flags[field].iloc[:start] == UNFLAGGED)
assert all(flags[field].iloc[end:] == UNFLAGGED)
- elif 'shift' in reshaper:
+ elif "shift" in reshaper:
if reshaper == "nshift":
exp = [False, False, False, False, True, False, False, False, False]
elif reshaper == "fshift":
@@ -145,58 +174,144 @@ def test_harmSingleVarIntermediateFlagging(data, reshaper):
elif reshaper == "bshift":
exp = [False, False, False, False, False, True, False, False, False]
else:
- raise NotImplementedError('untested test case')
+ raise NotImplementedError("untested test case")
flagged = flags[field] > UNFLAGGED
assert all(flagged == exp)
- elif reshaper == 'interpolation':
- pytest.skip('no testcase for interpolation')
+ elif reshaper == "interpolation":
+ pytest.skip("no testcase for interpolation")
else:
- raise NotImplementedError('untested test case')
+ raise NotImplementedError("untested test case")
@pytest.mark.parametrize(
- 'params, expected',
+ "params, expected",
[
- (("nagg", "15Min"), pd.Series(data=[-87.5, -25.0, 0.0, 37.5, 50.0], index=pd.date_range("2011-01-01 00:00:00", "2011-01-01 01:00:00", freq="15min"))),
- (("nagg", "30Min"), pd.Series(data=[-87.5, -25.0, 87.5], index=pd.date_range("2011-01-01 00:00:00", "2011-01-01 01:00:00", freq="30min"))),
- (("bagg", "15Min"), pd.Series(data=[-50.0, -37.5, -37.5, 12.5, 37.5, 50.0], index=pd.date_range("2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15min"))),
- (("bagg", "30Min"), pd.Series(data=[-50.0, -75.0, 50.0, 50.0], index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30min"))),
- ])
+ (
+ ("nagg", "15Min"),
+ pd.Series(
+ data=[-87.5, -25.0, 0.0, 37.5, 50.0],
+ index=pd.date_range(
+ "2011-01-01 00:00:00", "2011-01-01 01:00:00", freq="15min"
+ ),
+ ),
+ ),
+ (
+ ("nagg", "30Min"),
+ pd.Series(
+ data=[-87.5, -25.0, 87.5],
+ index=pd.date_range(
+ "2011-01-01 00:00:00", "2011-01-01 01:00:00", freq="30min"
+ ),
+ ),
+ ),
+ (
+ ("bagg", "15Min"),
+ pd.Series(
+ data=[-50.0, -37.5, -37.5, 12.5, 37.5, 50.0],
+ index=pd.date_range(
+ "2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15min"
+ ),
+ ),
+ ),
+ (
+ ("bagg", "30Min"),
+ pd.Series(
+ data=[-50.0, -75.0, 50.0, 50.0],
+ index=pd.date_range(
+ "2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30min"
+ ),
+ ),
+ ),
+ ],
+)
def test_harmSingleVarInterpolationAgg(data, params, expected):
flags = initFlagsLike(data)
- field = 'data'
+ field = "data"
pre_data = data.copy()
pre_flaggger = flags.copy()
method, freq = params
- data_harm, flags_harm = aggregate(data, field, flags, freq, value_func=np.sum, method=method)
+ data_harm, flags_harm = aggregate(
+ data, field, flags, freq, value_func=np.sum, method=method
+ )
checkDataFlagsInvariants(data_harm, flags_harm, field, identical=True)
assert data_harm[field].index.freq == pd.Timedelta(freq)
assert data_harm[field].equals(expected)
- data_deharm, flags_deharm = mapToOriginal(data_harm, "data", flags_harm, method="inverse_" + method)
+ data_deharm, flags_deharm = mapToOriginal(
+ data_harm, "data", flags_harm, method="inverse_" + method
+ )
checkDataFlagsInvariants(data_harm, flags_harm, field, identical=True)
assert data_deharm[field].equals(pre_data[field])
assert flags_deharm[field].equals(pre_flaggger[field])
@pytest.mark.parametrize(
- 'params, expected',
+ "params, expected",
[
- (("bshift", "15Min"), pd.Series(data=[-50.0, -37.5, -25.0, 12.5, 37.5, 50.0], index=pd.date_range("2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15Min"))),
- (("fshift", "15Min"), pd.Series(data=[np.nan, -37.5, -25.0, 0.0, 37.5, 50.0], index=pd.date_range("2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15Min"))),
- (("nshift", "15min"), pd.Series(data=[np.nan, -37.5, -25.0, 12.5, 37.5, 50.0], index=pd.date_range("2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15Min"))),
- (("bshift", "30Min"), pd.Series(data=[-50.0, -37.5, 12.5, 50.0], index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"))),
- (("fshift", "30Min"), pd.Series(data=[np.nan, -37.5, 0.0, 50.0], index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"))),
- (("nshift", "30min"), pd.Series(data=[np.nan, -37.5, 12.5, 50.0], index=pd.date_range("2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"))),
- ])
+ (
+ ("bshift", "15Min"),
+ pd.Series(
+ data=[-50.0, -37.5, -25.0, 12.5, 37.5, 50.0],
+ index=pd.date_range(
+ "2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15Min"
+ ),
+ ),
+ ),
+ (
+ ("fshift", "15Min"),
+ pd.Series(
+ data=[np.nan, -37.5, -25.0, 0.0, 37.5, 50.0],
+ index=pd.date_range(
+ "2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15Min"
+ ),
+ ),
+ ),
+ (
+ ("nshift", "15min"),
+ pd.Series(
+ data=[np.nan, -37.5, -25.0, 12.5, 37.5, 50.0],
+ index=pd.date_range(
+ "2010-12-31 23:45:00", "2011-01-01 01:00:00", freq="15Min"
+ ),
+ ),
+ ),
+ (
+ ("bshift", "30Min"),
+ pd.Series(
+ data=[-50.0, -37.5, 12.5, 50.0],
+ index=pd.date_range(
+ "2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"
+ ),
+ ),
+ ),
+ (
+ ("fshift", "30Min"),
+ pd.Series(
+ data=[np.nan, -37.5, 0.0, 50.0],
+ index=pd.date_range(
+ "2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"
+ ),
+ ),
+ ),
+ (
+ ("nshift", "30min"),
+ pd.Series(
+ data=[np.nan, -37.5, 12.5, 50.0],
+ index=pd.date_range(
+ "2010-12-31 23:30:00", "2011-01-01 01:00:00", freq="30Min"
+ ),
+ ),
+ ),
+ ],
+)
def test_harmSingleVarInterpolationShift(data, params, expected):
flags = initFlagsLike(data)
- field = 'data'
+ field = "data"
pre_data = data.copy()
pre_flags = flags.copy()
method, freq = params
@@ -204,8 +319,8 @@ def test_harmSingleVarInterpolationShift(data, params, expected):
data_harm, flags_harm = shift(data, field, flags, freq, method=method)
assert data_harm[field].equals(expected)
- data_deharm, flags_deharm = mapToOriginal(data_harm, "data", flags_harm, method="inverse_" + method)
+ data_deharm, flags_deharm = mapToOriginal(
+ data_harm, "data", flags_harm, method="inverse_" + method
+ )
assert data_deharm[field].equals(pre_data[field])
assert flags_deharm[field].equals(pre_flags[field])
-
-
diff --git a/tests/funcs/test_modelling.py b/tests/funcs/test_modelling.py
index 5bfdfba88..76b2211e2 100644
--- a/tests/funcs/test_modelling.py
+++ b/tests/funcs/test_modelling.py
@@ -16,7 +16,9 @@ from tests.fixtures import *
@pytest.mark.parametrize("dat", [pytest.lazy_fixture("course_2")])
def test_modelling_polyFit_forRegular(dat):
- data, _ = dat(freq="10min", periods=30, initial_level=0, final_level=100, out_val=-100)
+ data, _ = dat(
+ freq="10min", periods=30, initial_level=0, final_level=100, out_val=-100
+ )
# add some nice sine distortion
data = data + 10 * np.sin(np.arange(0, len(data.indexes[0])))
data = dios.DictOfSeries(data)
@@ -24,22 +26,48 @@ def test_modelling_polyFit_forRegular(dat):
result1, _ = calculatePolynomialResidues(data, "data", flags, 11, 2, numba=False)
result2, _ = calculatePolynomialResidues(data, "data", flags, 11, 2, numba=True)
assert (result1["data"] - result2["data"]).abs().max() < 10 ** -10
- result3, _ = calculatePolynomialResidues(data, "data", flags, "110min", 2, numba=False)
+ result3, _ = calculatePolynomialResidues(
+ data, "data", flags, "110min", 2, numba=False
+ )
assert result3["data"].equals(result1["data"])
- result4, _ = calculatePolynomialResidues(data, "data", flags, 11, 2, numba=True, min_periods=11)
+ result4, _ = calculatePolynomialResidues(
+ data, "data", flags, 11, 2, numba=True, min_periods=11
+ )
assert (result4["data"] - result2["data"]).abs().max() < 10 ** -10
data.iloc[13:16] = np.nan
- result5, _ = calculatePolynomialResidues(data, "data", flags, 11, 2, numba=True, min_periods=9)
+ result5, _ = calculatePolynomialResidues(
+ data, "data", flags, 11, 2, numba=True, min_periods=9
+ )
assert result5["data"].iloc[10:19].isna().all()
@pytest.mark.parametrize("dat", [pytest.lazy_fixture("course_2")])
def test_modelling_rollingMean_forRegular(dat):
- data, _ = dat(freq="10min", periods=30, initial_level=0, final_level=100, out_val=-100)
+ data, _ = dat(
+ freq="10min", periods=30, initial_level=0, final_level=100, out_val=-100
+ )
data = dios.DictOfSeries(data)
flags = initFlagsLike(data)
- calculateRollingResidues(data, "data", flags, 5, func=np.mean, eval_flags=True, min_periods=0, center=True)
- calculateRollingResidues(data, "data", flags, 5, func=np.mean, eval_flags=True, min_periods=0, center=False)
+ calculateRollingResidues(
+ data,
+ "data",
+ flags,
+ 5,
+ func=np.mean,
+ eval_flags=True,
+ min_periods=0,
+ center=True,
+ )
+ calculateRollingResidues(
+ data,
+ "data",
+ flags,
+ 5,
+ func=np.mean,
+ eval_flags=True,
+ min_periods=0,
+ center=False,
+ )
@pytest.mark.parametrize("dat", [pytest.lazy_fixture("course_1")])
@@ -52,20 +80,26 @@ def test_modelling_mask(dat):
# set flags everywhere to test unflagging
flags[:, field] = BAD
- common = dict(data=data, field=field, flags=flags, mode='periodic')
- data_seasonal, flags_seasonal = mask(**common, period_start="20:00", period_end="40:00", include_bounds=False)
+ common = dict(data=data, field=field, flags=flags, mode="periodic")
+ data_seasonal, flags_seasonal = mask(
+ **common, period_start="20:00", period_end="40:00", include_bounds=False
+ )
flagscol = flags_seasonal[field]
m = (20 <= flagscol.index.minute) & (flagscol.index.minute <= 40)
assert all(flags_seasonal[field][m] == UNFLAGGED)
assert all(data_seasonal[field][m].isna())
- data_seasonal, flags_seasonal = mask(**common, period_start="15:00:00", period_end="02:00:00")
+ data_seasonal, flags_seasonal = mask(
+ **common, period_start="15:00:00", period_end="02:00:00"
+ )
flagscol = flags_seasonal[field]
m = (15 <= flagscol.index.hour) & (flagscol.index.hour <= 2)
assert all(flags_seasonal[field][m] == UNFLAGGED)
assert all(data_seasonal[field][m].isna())
- data_seasonal, flags_seasonal = mask(**common, period_start="03T00:00:00", period_end="10T00:00:00")
+ data_seasonal, flags_seasonal = mask(
+ **common, period_start="03T00:00:00", period_end="10T00:00:00"
+ )
flagscol = flags_seasonal[field]
m = (3 <= flagscol.index.hour) & (flagscol.index.hour <= 10)
assert all(flags_seasonal[field][m] == UNFLAGGED)
@@ -75,7 +109,9 @@ def test_modelling_mask(dat):
mask_ser[::5] = True
data["mask_ser"] = mask_ser
flags = initFlagsLike(data)
- data_masked, flags_masked = mask(data, "data", flags, mode='mask_var', mask_var="mask_ser")
+ data_masked, flags_masked = mask(
+ data, "data", flags, mode="mask_var", mask_var="mask_ser"
+ )
m = mask_ser
assert all(flags_masked[field][m] == UNFLAGGED)
assert all(data_masked[field][m].isna())
diff --git a/tests/funcs/test_pattern_rec.py b/tests/funcs/test_pattern_rec.py
index 1cd7b7b4d..d8d67e324 100644
--- a/tests/funcs/test_pattern_rec.py
+++ b/tests/funcs/test_pattern_rec.py
@@ -21,30 +21,34 @@ def field(data):
return data.columns[0]
-@pytest.mark.skip(reason='faulty implementation - will get fixed by GL-MR191')
+@pytest.mark.skip(reason="faulty implementation - will get fixed by GL-MR191")
def test_flagPattern_wavelet():
- data = pd.Series(0, index=pd.date_range(start="2000", end='2001', freq='1d'))
+ data = pd.Series(0, index=pd.date_range(start="2000", end="2001", freq="1d"))
data.iloc[2:4] = 7
pattern = data.iloc[1:6]
data = dios.DictOfSeries(dict(data=data, pattern_data=pattern))
- flags = initFlagsLike(data, name='data')
- data, flags = flagPatternByDTW(data, "data", flags, ref_field="pattern_data", flag=BAD)
+ flags = initFlagsLike(data, name="data")
+ data, flags = flagPatternByDTW(
+ data, "data", flags, ref_field="pattern_data", flag=BAD
+ )
assert all(flags["data"][1:6])
assert any(flags["data"][:1])
assert any(flags["data"][7:])
-@pytest.mark.skip(reason='faulty implementation - will get fixed by GL-MR191')
+@pytest.mark.skip(reason="faulty implementation - will get fixed by GL-MR191")
def test_flagPattern_dtw():
- data = pd.Series(0, index=pd.date_range(start="2000", end='2001', freq='1d'))
+ data = pd.Series(0, index=pd.date_range(start="2000", end="2001", freq="1d"))
data.iloc[2:4] = 7
pattern = data.iloc[1:6]
data = dios.DictOfSeries(dict(data=data, pattern_data=pattern))
- flags = initFlagsLike(data, name='data')
- data, flags = flagPatternByWavelet(data, "data", flags, ref_field="pattern_data", flag=BAD)
+ flags = initFlagsLike(data, name="data")
+ data, flags = flagPatternByWavelet(
+ data, "data", flags, ref_field="pattern_data", flag=BAD
+ )
assert all(flags["data"][1:6])
assert any(flags["data"][:1])
diff --git a/tests/funcs/test_proc_functions.py b/tests/funcs/test_proc_functions.py
index cfcd5bcf6..22145a1fb 100644
--- a/tests/funcs/test_proc_functions.py
+++ b/tests/funcs/test_proc_functions.py
@@ -10,7 +10,11 @@ from saqc.constants import *
from saqc.core import initFlagsLike
from saqc.funcs.transformation import transform
from saqc.funcs.drift import correctOffset
-from saqc.funcs.interpolation import interpolateByRolling, interpolateInvalid, interpolateIndex
+from saqc.funcs.interpolation import (
+ interpolateByRolling,
+ interpolateInvalid,
+ interpolateIndex,
+)
from saqc.funcs.resampling import resample
from saqc.lib.ts_operators import linearInterpolation, polynomialInterpolation
@@ -23,13 +27,27 @@ def test_rollingInterpolateMissing(course_5):
data = dios.DictOfSeries(data)
flags = initFlagsLike(data)
dataInt, *_ = interpolateByRolling(
- data, field, flags, 3, func=np.median, center=True, min_periods=0, interpol_flag=UNFLAGGED
+ data,
+ field,
+ flags,
+ 3,
+ func=np.median,
+ center=True,
+ min_periods=0,
+ interpol_flag=UNFLAGGED,
)
# import pdb
# pdb.set_trace()
assert dataInt[field][characteristics["missing"]].notna().all()
dataInt, *_ = interpolateByRolling(
- data, field, flags, 3, func=np.nanmean, center=False, min_periods=3, interpol_flag=UNFLAGGED
+ data,
+ field,
+ flags,
+ 3,
+ func=np.nanmean,
+ center=False,
+ min_periods=3,
+ interpol_flag=UNFLAGGED,
)
assert dataInt[field][characteristics["missing"]].isna().all()
@@ -44,9 +62,15 @@ def test_interpolateMissing(course_5):
assert dataLin[field][characteristics["missing"]].notna().all()
assert dataPoly[field][characteristics["missing"]].notna().all()
data, characteristics = course_5(periods=10, nan_slice=[5, 6, 7])
- dataLin1, *_ = interpolateInvalid(data, field, flags, method="linear", inter_limit=2)
- dataLin2, *_ = interpolateInvalid(data, field, flags, method="linear", inter_limit=3)
- dataLin3, *_ = interpolateInvalid(data, field, flags, method="linear", inter_limit=4)
+ dataLin1, *_ = interpolateInvalid(
+ data, field, flags, method="linear", inter_limit=2
+ )
+ dataLin2, *_ = interpolateInvalid(
+ data, field, flags, method="linear", inter_limit=3
+ )
+ dataLin3, *_ = interpolateInvalid(
+ data, field, flags, method="linear", inter_limit=4
+ )
assert dataLin1[field][characteristics["missing"]].isna().all()
assert dataLin2[field][characteristics["missing"]].isna().all()
assert dataLin3[field][characteristics["missing"]].notna().all()
@@ -59,20 +83,35 @@ def test_transform(course_5):
flags = initFlagsLike(data)
data1, *_ = transform(data, field, flags, func=linearInterpolation)
assert data1[field][characteristics["missing"]].isna().all()
- data1, *_ = transform(data, field, flags, func=lambda x: linearInterpolation(x, inter_limit=3))
+ data1, *_ = transform(
+ data, field, flags, func=lambda x: linearInterpolation(x, inter_limit=3)
+ )
assert data1[field][characteristics["missing"]].notna().all()
data1, *_ = transform(
- data, field, flags, func=lambda x: polynomialInterpolation(x, inter_limit=3, inter_order=3)
+ data,
+ field,
+ flags,
+ func=lambda x: polynomialInterpolation(x, inter_limit=3, inter_order=3),
)
assert data1[field][characteristics["missing"]].notna().all()
def test_resample(course_5):
- data, characteristics = course_5(freq="1min", periods=30, nan_slice=[1, 11, 12, 22, 24, 26])
+ data, characteristics = course_5(
+ freq="1min", periods=30, nan_slice=[1, 11, 12, 22, 24, 26]
+ )
field = data.columns[0]
data = dios.DictOfSeries(data)
flags = initFlagsLike(data)
- data1, *_ = resample(data, field, flags, "10min", np.mean, max_invalid_total_d=2, max_invalid_consec_d=1)
+ data1, *_ = resample(
+ data,
+ field,
+ flags,
+ "10min",
+ np.mean,
+ max_invalid_total_d=2,
+ max_invalid_consec_d=1,
+ )
assert ~np.isnan(data1[field].iloc[0])
assert np.isnan(data1[field].iloc[1])
assert np.isnan(data1[field].iloc[2])
@@ -81,18 +120,19 @@ def test_resample(course_5):
def test_interpolateGrid(course_5, course_3):
data, _ = course_5()
data_grid, characteristics = course_3()
- data['grid'] = data_grid.to_df()
+ data["grid"] = data_grid.to_df()
# data = dios.DictOfSeries(data)
flags = initFlagsLike(data)
- dataInt, *_ = interpolateIndex(data, 'data', flags, '1h', 'time', grid_field='grid', inter_limit=10)
+ dataInt, *_ = interpolateIndex(
+ data, "data", flags, "1h", "time", grid_field="grid", inter_limit=10
+ )
def test_offsetCorrecture():
- data = pd.Series(0, index=pd.date_range('2000', freq='1d', periods=100), name='dat')
+ data = pd.Series(0, index=pd.date_range("2000", freq="1d", periods=100), name="dat")
data.iloc[30:40] = -100
data.iloc[70:80] = 100
data = dios.DictOfSeries(data)
flags = initFlagsLike(data)
- data, _ = correctOffset(data, 'dat', flags, 40, 20, '3d', 1)
+ data, _ = correctOffset(data, "dat", flags, 40, 20, "3d", 1)
assert (data == 0).all()[0]
-
diff --git a/tests/funcs/test_spikes_detection.py b/tests/funcs/test_spikes_detection.py
index 9481d7eb0..95132dd2f 100644
--- a/tests/funcs/test_spikes_detection.py
+++ b/tests/funcs/test_spikes_detection.py
@@ -40,7 +40,9 @@ def test_flagSpikesBasic(spiky_data):
data = spiky_data[0]
field, *_ = data.columns
flags = initFlagsLike(data)
- data, flags_result = flagOffset(data, field, flags, thresh=60, tolerance=10, window="20min", flag=BAD)
+ data, flags_result = flagOffset(
+ data, field, flags, thresh=60, tolerance=10, window="20min", flag=BAD
+ )
flag_result = flags_result[field]
test_sum = (flag_result[spiky_data[1]] == BAD).sum()
assert test_sum == len(spiky_data[1])
@@ -61,8 +63,14 @@ def test_flagSpikesLimitRaise(dat):
field, *_ = data.columns
flags = initFlagsLike(data)
_, flags_result = flagRaise(
- data, field, flags,
- thresh=2, intended_freq="10min", raise_window="20min", numba_boost=False, flag=BAD
+ data,
+ field,
+ flags,
+ thresh=2,
+ intended_freq="10min",
+ raise_window="20min",
+ numba_boost=False,
+ flag=BAD,
)
assert np.all(flags_result[field][characteristics["raise"]] > UNFLAGGED)
assert not np.any(flags_result[field][characteristics["return"]] > UNFLAGGED)
@@ -72,8 +80,12 @@ def test_flagSpikesLimitRaise(dat):
# see test/functs/fixtures.py for the 'course_N'
@pytest.mark.parametrize("dat", [pytest.lazy_fixture("course_3")])
def test_flagMultivarScores(dat):
- data1, characteristics = dat(periods=1000, initial_level=5, final_level=15, out_val=50)
- data2, characteristics = dat(periods=1000, initial_level=20, final_level=1, out_val=30)
+ data1, characteristics = dat(
+ periods=1000, initial_level=5, final_level=15, out_val=50
+ )
+ data2, characteristics = dat(
+ periods=1000, initial_level=20, final_level=1, out_val=30
+ )
field = "dummy"
fields = ["data1", "data2"]
s1, s2 = data1.squeeze(), data2.squeeze()
@@ -82,7 +94,14 @@ def test_flagMultivarScores(dat):
data = dios.DictOfSeries([s1, s2], columns=["data1", "data2"])
flags = initFlagsLike(data)
_, flags_result = flagMVScores(
- data, field, flags, fields=fields, trafo=np.log, iter_start=0.95, n_neighbors=10, flag=BAD
+ data,
+ field,
+ flags,
+ fields=fields,
+ trafo=np.log,
+ iter_start=0.95,
+ n_neighbors=10,
+ flag=BAD,
)
for field in fields:
isflagged = flags_result[field] > UNFLAGGED
@@ -94,12 +113,16 @@ def test_flagMultivarScores(dat):
@pytest.mark.parametrize("dat", [pytest.lazy_fixture("course_3")])
def test_grubbs(dat):
data, char_dict = dat(
- freq="10min", periods=45,
- initial_level=0, final_level=0,
- crowd_size=1, crowd_spacing=3,
+ freq="10min",
+ periods=45,
+ initial_level=0,
+ final_level=0,
+ crowd_size=1,
+ crowd_spacing=3,
out_val=-10,
)
flags = initFlagsLike(data)
- data, result_flags = flagByGrubbs(data, "data", flags, winsz=20, min_periods=15, flag=BAD)
+ data, result_flags = flagByGrubbs(
+ data, "data", flags, winsz=20, min_periods=15, flag=BAD
+ )
assert np.all(result_flags["data"][char_dict["drop"]] > UNFLAGGED)
-
diff --git a/tests/fuzzy/lib.py b/tests/fuzzy/lib.py
index b08bb65d8..c0cf16fe6 100644
--- a/tests/fuzzy/lib.py
+++ b/tests/fuzzy/lib.py
@@ -44,15 +44,14 @@ def dioses(draw, min_cols=1):
# the integer->float->integer type conversion in _maskData/_unmaskData.
cols = draw(lists(columnNames(), unique=True, min_size=min_cols))
- columns = {
- c: draw(dataSeries(min_size=3))
- for c in cols
- }
+ columns = {c: draw(dataSeries(min_size=3)) for c in cols}
return dios.DictOfSeries(columns)
@composite
-def dataSeries(draw, min_size=0, max_size=100, dtypes=("float32", "float64", "int32", "int64")):
+def dataSeries(
+ draw, min_size=0, max_size=100, dtypes=("float32", "float64", "int32", "int64")
+):
if np.isscalar(dtypes):
dtypes = (dtypes,)
@@ -83,7 +82,9 @@ def flagses(draw, data):
"""
flags = initFlagsLike(data)
for col, srs in data.items():
- loc_st = lists(sampled_from(sorted(srs.index)), unique=True, max_size=len(srs) - 1)
+ loc_st = lists(
+ sampled_from(sorted(srs.index)), unique=True, max_size=len(srs) - 1
+ )
flags[draw(loc_st), col] = BAD
return flags
@@ -135,14 +136,14 @@ def functionKwargs(draw, func: SaQCFunction):
data = draw(dioses())
field = draw(sampled_from(sorted(data.columns)))
- kwargs = {
- "data": data,
- "field": field,
- "flags": draw(flagses(data))
- }
+ kwargs = {"data": data, "field": field, "flags": draw(flagses(data))}
- column_name_strategy = lambda _: sampled_from(sorted(c for c in data.columns if c != field))
- interger_window_strategy = lambda _: integers(min_value=1, max_value=len(data[field]) - 1)
+ column_name_strategy = lambda _: sampled_from(
+ sorted(c for c in data.columns if c != field)
+ )
+ interger_window_strategy = lambda _: integers(
+ min_value=1, max_value=len(data[field]) - 1
+ )
register_type_strategy(FreqString, frequencyStrings)
register_type_strategy(ColumnName, column_name_strategy)
diff --git a/tests/fuzzy/test_functions.py b/tests/fuzzy/test_functions.py
index 63b8a8b27..1550d375c 100644
--- a/tests/fuzzy/test_functions.py
+++ b/tests/fuzzy/test_functions.py
@@ -38,6 +38,7 @@ def test_breaks_flagMissing():
# constants
# ---------
+
def test_constats_flagConstats():
callWontBreak("constants.flagConstants")
@@ -49,6 +50,7 @@ def test_constants_flagByVariance():
# flagtools
# ---------
+
def test_flagtools_clearFlags():
callWontBreak("flagtools.clearFlags")
diff --git a/tests/fuzzy/test_masking.py b/tests/fuzzy/test_masking.py
index ba897adc0..d0875906f 100644
--- a/tests/fuzzy/test_masking.py
+++ b/tests/fuzzy/test_masking.py
@@ -23,7 +23,9 @@ def test_maskingMasksData(data_field_flags):
test if flagged values are replaced by np.nan
"""
data_in, field, flags = data_field_flags
- data_masked, mask = _maskData(data_in, flags, columns=[field], thresh=UNFLAGGED) # thresh UNFLAGGED | np.inf
+ data_masked, mask = _maskData(
+ data_in, flags, columns=[field], thresh=UNFLAGGED
+ ) # thresh UNFLAGGED | np.inf
assert data_masked.loc[mask[field], field].isna().all()
assert (flags[field][mask[field]] > UNFLAGGED).all()
@@ -31,7 +33,7 @@ def test_maskingMasksData(data_field_flags):
@settings(max_examples=MAX_EXAMPLES, deadline=None)
@given(data_field_flags=dataFieldFlags())
def test_dataMutationPreventsUnmasking(data_field_flags):
- """ test if (un)masking works as expected on data-changes.
+ """test if (un)masking works as expected on data-changes.
if `data` is mutated after `_maskData`, `_unmaskData` should be a no-op
"""
@@ -41,10 +43,14 @@ def test_dataMutationPreventsUnmasking(data_field_flags):
data_masked, mask = _maskData(data_in, flags, columns=[field], thresh=UNFLAGGED)
state = CallState(
func=None,
- data=data_in, flags=flags, field=field,
- args=None, kwargs=None,
- masking="field", mthresh=UNFLAGGED,
- mask=mask
+ data=data_in,
+ flags=flags,
+ field=field,
+ args=None,
+ kwargs=None,
+ masking="field",
+ mthresh=UNFLAGGED,
+ mask=mask,
)
data_masked[field] = filler
@@ -55,7 +61,7 @@ def test_dataMutationPreventsUnmasking(data_field_flags):
@settings(max_examples=MAX_EXAMPLES, deadline=None)
@given(data_field_flags=dataFieldFlags())
def test_flagsMutationPreventsUnmasking(data_field_flags):
- """ test if (un)masking works as expected on flags-changes.
+ """test if (un)masking works as expected on flags-changes.
if `flags` is mutated after `_maskData`, `_unmaskData` should be a no-op
"""
@@ -63,10 +69,14 @@ def test_flagsMutationPreventsUnmasking(data_field_flags):
data_masked, mask = _maskData(data_in, flags, columns=[field], thresh=UNFLAGGED)
state = CallState(
func=None,
- data=data_in, flags=flags, field=field,
- args=None, kwargs=None,
- masking="field", mthresh=UNFLAGGED,
- mask=mask
+ data=data_in,
+ flags=flags,
+ field=field,
+ args=None,
+ kwargs=None,
+ masking="field",
+ mthresh=UNFLAGGED,
+ mask=mask,
)
flags[:, field] = UNFLAGGED
data_out = _unmaskData(data_masked, state)
@@ -76,7 +86,7 @@ def test_flagsMutationPreventsUnmasking(data_field_flags):
@settings(max_examples=MAX_EXAMPLES, deadline=None)
@given(data_field_flags=dataFieldFlags())
def test_reshapingPreventsUnmasking(data_field_flags):
- """ test if (un)masking works as expected on index-changes.
+ """test if (un)masking works as expected on index-changes.
If the index of data (and flags) change in the func, the unmasking,
should not reapply original data, instead take the new data (and flags) as is.
@@ -88,14 +98,18 @@ def test_reshapingPreventsUnmasking(data_field_flags):
data_masked, mask = _maskData(data_in, flags, columns=[field], thresh=UNFLAGGED)
state = CallState(
func=None,
- data=data_in, flags=flags, field=field,
- args=None, kwargs=None,
- masking="field", mthresh=UNFLAGGED,
- mask=mask
+ data=data_in,
+ flags=flags,
+ field=field,
+ args=None,
+ kwargs=None,
+ masking="field",
+ mthresh=UNFLAGGED,
+ mask=mask,
)
# mutate indexes of `data` and `flags`
index = data_masked[field].index.to_series()
- index.iloc[-len(data_masked[field])//2:] += pd.Timedelta("7.5Min")
+ index.iloc[-len(data_masked[field]) // 2 :] += pd.Timedelta("7.5Min")
data_masked[field] = pd.Series(data=filler, index=index)
fflags = flags[field]
@@ -116,10 +130,14 @@ def test_unmaskingInvertsMasking(data_field_flags):
data_masked, mask = _maskData(data_in, flags, columns=[field], thresh=UNFLAGGED)
state = CallState(
func=None,
- data=data_in, flags=flags, field=field,
- args=None, kwargs=None,
- masking="field", mthresh=UNFLAGGED,
- mask=mask
+ data=data_in,
+ flags=flags,
+ field=field,
+ args=None,
+ kwargs=None,
+ masking="field",
+ mthresh=UNFLAGGED,
+ mask=mask,
)
data_out = _unmaskData(data_masked, state)
assert pd.DataFrame.equals(
diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py
index e88a90ab3..019ab4f37 100644
--- a/tests/integration/test_integration.py
+++ b/tests/integration/test_integration.py
@@ -7,13 +7,18 @@ def test__main__py():
import saqc.__main__
# if not run from project root
- projpath = os.path.dirname(saqc.__file__) + '/../'
+ projpath = os.path.dirname(saqc.__file__) + "/../"
runner = CliRunner()
result = runner.invoke(
- saqc.__main__.main, [
- '--config', projpath + 'ressources/data/config_ci.csv',
- '--data', projpath + 'ressources/data/data.csv',
- '--outfile', '/tmp/test.csv', # the filesystem temp dir
- ])
+ saqc.__main__.main,
+ [
+ "--config",
+ projpath + "ressources/data/config_ci.csv",
+ "--data",
+ projpath + "ressources/data/data.csv",
+ "--outfile",
+ "/tmp/test.csv", # the filesystem temp dir
+ ],
+ )
assert result.exit_code == 0, result.output
diff --git a/tests/lib/test_rolling.py b/tests/lib/test_rolling.py
index b7245b3b0..6968a1431 100644
--- a/tests/lib/test_rolling.py
+++ b/tests/lib/test_rolling.py
@@ -4,12 +4,26 @@ from saqc.lib.rolling import customRoller, Rolling
import pandas as pd
import numpy as np
-FUNCTS = ['count', 'sum', 'mean', 'median', 'var', 'std', 'min', 'max', 'corr', 'cov', 'skew', 'kurt', ]
-
-OTHA = ['apply',
- 'aggregate', # needs param func eg. func='min'
- 'quantile', # needs param quantile=0.5 (0<=q<=1)
- ]
+FUNCTS = [
+ "count",
+ "sum",
+ "mean",
+ "median",
+ "var",
+ "std",
+ "min",
+ "max",
+ "corr",
+ "cov",
+ "skew",
+ "kurt",
+]
+
+OTHA = [
+ "apply",
+ "aggregate", # needs param func eg. func='min'
+ "quantile", # needs param quantile=0.5 (0<=q<=1)
+]
@pytest.fixture
@@ -18,10 +32,12 @@ def data():
def data_():
- s1 = pd.Series(1., index=pd.date_range("1999/12", periods=12, freq='1M') + pd.Timedelta('1d'))
- s2 = pd.Series(1., index=pd.date_range('2000/05/15', periods=8, freq='1d'))
+ s1 = pd.Series(
+ 1.0, index=pd.date_range("1999/12", periods=12, freq="1M") + pd.Timedelta("1d")
+ )
+ s2 = pd.Series(1.0, index=pd.date_range("2000/05/15", periods=8, freq="1d"))
s = pd.concat([s1, s2]).sort_index()
- s.name = 's'
+ s.name = "s"
s[15] = np.nan
return s
@@ -46,10 +62,12 @@ def make_dt_kws():
l = []
n = [0, 1, 2, 10, 32, 70, 120]
mp = list(range(len_s))
- for closed in ['right', 'both', 'neither', 'left']:
+ for closed in ["right", "both", "neither", "left"]:
for window in n:
for min_periods in [None] + mp:
- l.append(dict(window=f'{window}d', min_periods=min_periods, closed=closed))
+ l.append(
+ dict(window=f"{window}d", min_periods=min_periods, closed=closed)
+ )
return l
@@ -65,9 +83,9 @@ def check_series(result, expected):
def print_diff(s, result, expected):
df = pd.DataFrame()
- df['s'] = s
- df['exp'] = expected
- df['res'] = result
+ df["s"] = s
+ df["exp"] = expected
+ df["res"] = result
print(df)
@@ -75,7 +93,7 @@ def call_rolling_function(roller, func):
if isinstance(func, str):
return getattr(roller, func)()
else:
- return getattr(roller, 'apply')(func)
+ return getattr(roller, "apply")(func)
@pytest.mark.parametrize("kws", make_dt_kws(), ids=lambda x: str(x))
@@ -93,7 +111,7 @@ def test_pandas_conform_dt(data, kws, func):
except Exception as e1:
assert type(e0) == type(e1)
return
- assert False, 'pandas faild, but we succeed'
+ assert False, "pandas faild, but we succeed"
resR = customRoller(s, **kws)
result = call_rolling_function(resR, func)
@@ -119,7 +137,7 @@ def test_pandas_conform_num(data, kws, func):
except Exception as e1:
assert type(e0) == type(e1)
return
- assert False, 'pandas faild, but we succeed'
+ assert False, "pandas faild, but we succeed"
resR = customRoller(s, **kws)
result = call_rolling_function(resR, func)
@@ -145,7 +163,7 @@ def test_forward_dt(data, kws, func):
except Exception as e1:
assert type(e0) == type(e1)
return
- assert False, 'pandas faild, but we succeed'
+ assert False, "pandas faild, but we succeed"
resR = customRoller(s, forward=True, **kws)
result = call_rolling_function(resR, func)
@@ -171,7 +189,7 @@ def test_forward_num(data, kws, func):
except Exception as e1:
assert type(e0) == type(e1)
return
- assert False, 'pandas faild, but we succeed'
+ assert False, "pandas faild, but we succeed"
resR = customRoller(s, forward=True, **kws)
result = call_rolling_function(resR, func)
@@ -193,14 +211,14 @@ def dt_center_kws():
@pytest.mark.parametrize("kws", dt_center_kws(), ids=lambda x: str(x))
def test_centering_w_dtindex(kws):
print(kws)
- s = pd.Series(0., index=pd.date_range("2000", periods=10, freq='1H'))
+ s = pd.Series(0.0, index=pd.date_range("2000", periods=10, freq="1H"))
s[4:7] = 1
- w = kws.pop('window')
- mp = kws.pop('min_periods')
+ w = kws.pop("window")
+ mp = kws.pop("min_periods")
pd_kw = dict(window=w, center=True, min_periods=mp)
- our_kw = dict(window=f'{w}h', center=True, closed='both', min_periods=mp)
+ our_kw = dict(window=f"{w}h", center=True, closed="both", min_periods=mp)
expected = s.rolling(**pd_kw).sum()
result = customRoller(s, **our_kw).sum()
success = check_series(result, expected)
@@ -211,7 +229,7 @@ def test_centering_w_dtindex(kws):
w -= 1
mp -= 1
pd_kw = dict(window=w, center=True, min_periods=mp)
- our_kw = dict(window=f'{w}h', center=True, closed='neither', min_periods=mp)
+ our_kw = dict(window=f"{w}h", center=True, closed="neither", min_periods=mp)
expected = s.rolling(**pd_kw).sum()
result = customRoller(s, **our_kw).sum()
success = check_series(result, expected)
--
GitLab
From 3a99bc9ec2f18bcefaabc32546ab53b8b5069c93 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Wed, 14 Apr 2021 07:42:31 +0200
Subject: [PATCH 118/180] formatting
---
saqc/core/translator.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
index 33c93d9ad..771d30b9f 100644
--- a/saqc/core/translator.py
+++ b/saqc/core/translator.py
@@ -21,7 +21,6 @@ ForwardMap = Dict[ExternalFlag, float]
BackwardMap = Dict[float, ExternalFlag]
-
class Translator:
"""
This class provides the basic translation mechanism and should serve as
--
GitLab
From 968f99c3e0014ed4cad10ca4e6456fce73d20107 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Mon, 26 Apr 2021 17:57:28 +0200
Subject: [PATCH 119/180] fixed missing initial UNFLAGGED column on flags init
---
saqc/core/flags.py | 36 +++++++++++++++++++++---------------
1 file changed, 21 insertions(+), 15 deletions(-)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index 34b2fed23..ecbf9c84f 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -191,24 +191,21 @@ class Flags:
for k, item in data.items():
+ if not isinstance(k, str):
+ raise ValueError("column names must be string")
if k in result:
raise ValueError("raw_data must not have duplicate keys")
- # No, means no ! (copy)
- if isinstance(item, History) and not copy:
- result[k] = item
+ if isinstance(item, History):
+ result[k] = History(item, copy=True) if copy else item
continue
- if isinstance(item, pd.Series):
- item = item.to_frame(name=0)
- elif isinstance(item, History):
- pass
- else:
+ if not isinstance(item, pd.Series):
raise TypeError(
f"cannot init from {type(data.__name__)} of {type(item.__name__)}"
)
- result[k] = History(item, copy=copy)
+ result[k] = _simpleHist(item.index).append(item, force=True)
return result
@@ -327,7 +324,7 @@ class Flags:
return
if key not in self._data:
- self._data[key] = History()
+ self._data[key] = _simpleHist(value.index)
self._data[key].append(value, force=True)
self._cache.pop(key, None)
@@ -456,8 +453,8 @@ def initFlagsLike(
name : str, default None
Only respected if `reference` is of type ``pd.Series``.
The column name that is used for the Flags. If ``None``
- the name of the series itself is taken, if this is also
- `None`, a ValueError is raised.
+ the name of the series itself is taken, if it is unset,
+ a ValueError is raised.
Notes
-----
@@ -502,8 +499,17 @@ def initFlagsLike(
if not isinstance(item, (pd.Series, History)):
raise TypeError("items in reference must be of type pd.Series")
- item = pd.DataFrame(initial_value, index=item.index, columns=[0], dtype=float)
-
- result[k] = History(item)
+ result[k] = _simpleHist(item.index, initial_value)
return Flags(result)
+
+
+def _simpleHist(index, initial_value: float = UNFLAGGED) -> History:
+ """
+ Make a single columned History from an index and an initial value.
+
+ Notes
+ -----
+ For internal use only.
+ """
+ return History(pd.DataFrame(initial_value, index=index, columns=[0], dtype=float))
--
GitLab
From 5a37213c32d3c0ba474414538f0371ef88a11c36 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Mon, 26 Apr 2021 18:06:16 +0200
Subject: [PATCH 120/180] minor change
---
saqc/core/flags.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index ecbf9c84f..a37adf279 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -197,7 +197,7 @@ class Flags:
raise ValueError("raw_data must not have duplicate keys")
if isinstance(item, History):
- result[k] = History(item, copy=True) if copy else item
+ result[k] = item.copy() if copy else item
continue
if not isinstance(item, pd.Series):
--
GitLab
From 81f521d4e7ba6231e7590804588a6555a131b6c6 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Fri, 16 Apr 2021 08:56:59 +0200
Subject: [PATCH 121/180] PositionalTranslator: Integer Flags
---
saqc/core/translator.py | 4 ++--
tests/core/test_translator.py | 12 ++++++------
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
index 771d30b9f..b52001ea4 100644
--- a/saqc/core/translator.py
+++ b/saqc/core/translator.py
@@ -311,7 +311,7 @@ class PositionalTranslator(Translator):
for field in flags.columns:
# drop the first column (i.e. the '9')
fflags = pd.DataFrame(
- flags[field].apply(tuple).tolist(), index=flags[field].index
+ flags[field].astype(str).apply(tuple).tolist(), index=flags[field].index
).iloc[:, 1:]
tflags = super().forward(fflags.astype(int)).toFrame()
@@ -345,4 +345,4 @@ class PositionalTranslator(Translator):
)
# NOTE: work around the default first column history columns (see GL#182)
out[field] = "9" + tflags.str.slice(start=1)
- return pd.DataFrame(out)
+ return pd.DataFrame(out).fillna(-9999).astype(int)
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index 400b96a8e..ec164afa5 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -119,9 +119,9 @@ def test_positionalTranslator():
flags[2::3, "var1"] = BAD
tflags = translator.backward(flags, None) # type: ignore
- assert (tflags["var2"].dropna() == "9").all(axis=None)
- assert (tflags["var1"].iloc[1::3] == "9210").all(axis=None)
- assert (tflags["var1"].iloc[2::3] == "9002").all(axis=None)
+ assert (tflags["var2"].replace(-9999, np.nan).dropna() == 9).all(axis=None)
+ assert (tflags["var1"].iloc[1::3] == 9210).all(axis=None)
+ assert (tflags["var1"].iloc[2::3] == 9002).all(axis=None)
def test_positionalTranslatorIntegration():
@@ -135,7 +135,7 @@ def test_positionalTranslatorIntegration():
data, flags = saqc.getResult()
for field in flags.columns:
- assert flags[field].str.match("^9[012]*$").all()
+ assert flags[field].astype(str).str.match("^9[012]*$").all()
round_trip = translator.backward(translator.forward(flags), saqc._computed)
assert (flags.values == round_trip.values).all()
@@ -216,8 +216,8 @@ def test_positionalTranslationPreservesFlags():
translator = PositionalTranslator()
_, flags1 = saqc1.getResult(raw=True)
_, flags2 = saqc2.getResult(raw=True)
- tflags1 = translator.backward(flags1, saqc1._computed)
- tflags2 = translator.backward(flags2, saqc2._computed)
+ tflags1 = translator.backward(flags1, saqc1._computed).astype(str)
+ tflags2 = translator.backward(flags2, saqc2._computed).astype(str)
for k in flags2.columns:
expected = tflags1[k].str.slice(start=1) * 2
--
GitLab
From b569fd9f0b2bc4b7e5a6dbb7793ea3f2cf73c98e Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Fri, 16 Apr 2021 11:45:18 +0200
Subject: [PATCH 122/180] Flags: fix potentially missing init column
---
saqc/core/flags.py | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index ead06fffb..e98cc4170 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -5,7 +5,7 @@ from dios.dios import DictOfSeries
import pandas as pd
import dios
-from typing import Union, Dict, DefaultDict, Optional, Type, Tuple, Iterable
+from typing import Mapping, Union, Dict, DefaultDict, Optional, Type, Tuple, Iterable
from saqc.constants import *
from saqc.core.history import History
@@ -183,7 +183,8 @@ class Flags:
# have to much trouble.
self._cache = {}
- def _initFromRaw(self, data, copy) -> Dict[str, History]:
+ @staticmethod
+ def _initFromRaw(data: Mapping, copy: bool) -> Dict[str, History]:
"""
init from dict-like: keys are flag column, values become
initial columns of history(s).
@@ -211,7 +212,11 @@ class Flags:
f"cannot init from '{type(data).__name__}' of '{type(item).__name__}'"
)
- result[k] = History(item, copy=copy)
+ hist = History(
+ pd.DataFrame(UNFLAGGED, index=item.index, columns=[0], dtype=float),
+ copy=copy
+ )
+ result[k] = hist.append(History(item, copy=copy), force=True)
return result
@@ -470,9 +475,9 @@ def initFlagsLike(
Implementation detail:
The resulting Flags has not necessarily the exact same (inner) dimensions as the reference.
- This may happen, if the passed structure, already holds History objects. Those are
+ This may happen, if the passed structure already holds History objects. Those are
reduced 1D-DataFrame (1-column-History). Nevertheless the returned flags are perfectly suitable
- to be used in Saqc as flags container along with the passed reference structure (data).
+ to be used in SaQC as flags container along with the passed reference structure (data).
Returns
-------
--
GitLab
From 2272dfb49eb0311e2bc36d4c32197f7e2a99c2fa Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Fri, 16 Apr 2021 19:01:14 +0200
Subject: [PATCH 123/180] default arguments
---
saqc/core/lib.py | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/saqc/core/lib.py b/saqc/core/lib.py
index 85b00ec8c..ad288b086 100644
--- a/saqc/core/lib.py
+++ b/saqc/core/lib.py
@@ -7,11 +7,11 @@ from typing import Optional, Any
from typing_extensions import Literal
-@dataclass
class ColumnSelector:
- field: str
- target: str
- regex: bool
+ def __init__(self, field, target=None, regex=False):
+ self.field = field
+ self.target = target or field
+ self.regex = regex
# TODO: this seems obsolete
@@ -36,12 +36,16 @@ class ConfigController(APIController):
class SaQCFunction:
- def __init__(self, name, function, *args, **keywords):
+ def __init__(self, name = "unknown", function=lambda x: x, *args, **keywords):
self.name = name
self.func = function
self.args = args
self.keywords = keywords
+ @property
+ def __name__(self):
+ return self.name
+
def __repr__(self):
return f"{self.__class__.__name__}.{self.func.__name__}"
--
GitLab
From 67d278a0e8ca667e5ad08f3b053af5416df784f0 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Fri, 16 Apr 2021 19:01:43 +0200
Subject: [PATCH 124/180] moved FuncReturnT as it tends to circular imports
---
saqc/core/register.py | 3 ++-
saqc/lib/types.py | 3 ---
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/saqc/core/register.py b/saqc/core/register.py
index bad96aba5..26fa510ca 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -11,7 +11,7 @@ import warnings
from saqc.constants import *
from saqc.core.lib import SaQCFunction
from saqc.core.flags import initFlagsLike, Flags
-from saqc.lib.types import FuncReturnT
+
# NOTE:
# the global SaQC function store,
@@ -19,6 +19,7 @@ from saqc.lib.types import FuncReturnT
FUNC_MAP: Dict[str, SaQCFunction] = {}
MaskingStrT = Literal["all", "field", "none"]
+FuncReturnT = Tuple[dios.DictOfSeries, Flags]
@dataclasses.dataclass
diff --git a/saqc/lib/types.py b/saqc/lib/types.py
index a7e054f77..d13d01274 100644
--- a/saqc/lib/types.py
+++ b/saqc/lib/types.py
@@ -5,7 +5,6 @@ __all__ = [
"ArrayLike",
"PandasLike",
"DiosLikeT",
- "FuncReturnT",
"FreqString",
"ColumnName",
"IntegerWindow",
@@ -23,7 +22,6 @@ from typing_extensions import Protocol, Literal
import numpy as np
import pandas as pd
from dios import DictOfSeries
-from saqc.core.flags import Flags
from saqc.core.lib import SaQCFunction, ColumnSelector, APIController
@@ -32,7 +30,6 @@ ArrayLike = TypeVar("ArrayLike", np.ndarray, pd.Series, pd.DataFrame)
PandasLike = Union[pd.Series, pd.DataFrame, DictOfSeries]
DiosLikeT = Union[DictOfSeries, pd.DataFrame]
-FuncReturnT = Tuple[DictOfSeries, Flags]
ExternalFlag = Union[str, float, int]
--
GitLab
From 29a1c497e7454086b89dc2d36f6b97b014725b65 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Fri, 16 Apr 2021 19:04:26 +0200
Subject: [PATCH 125/180] added AGUMENTS
---
saqc/core/translator.py | 21 +++++++++++----------
1 file changed, 11 insertions(+), 10 deletions(-)
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
index b52001ea4..60e171d00 100644
--- a/saqc/core/translator.py
+++ b/saqc/core/translator.py
@@ -45,7 +45,11 @@ class Translator:
`UNFLAGGED`.
"""
- TO_MASK = True
+ # (internal) threshold flag above which values will be masked
+ TO_MASK: Union[float, bool] = True
+
+ # additional arguments and default values the translation scheme accpepts
+ ARGUMENTS: Dict[str, Any] = {}
def __init__(self, forward: ForwardMap, backward: Optional[BackwardMap] = None):
"""
@@ -78,7 +82,7 @@ class Translator:
flags: Union[Flags, pd.DataFrame], trans_map: Union[ForwardMap, BackwardMap]
) -> DictOfSeries:
"""
- Translate a given flag data structure to another one according to the
+ Translate a given flag data structure to another according to the
mapping given in `trans_map`
Parameters
@@ -122,7 +126,7 @@ class Translator:
def forward(self, flags: pd.DataFrame) -> Flags:
"""
- Translate from 'extrnal flags' to 'internal flags'
+ Translate from 'external flags' to 'internal flags'
Parameters
----------
@@ -179,23 +183,20 @@ class DmpTranslator(Translator):
the UFZ - Datamanagementportal
"""
+ ARGUMENTS = {"comment": "", "cause": ""}
+
_FORWARD: Dict[str, float] = {
"NIL": UNFLAGGED,
"OK": GOOD,
"DOUBTFUL": DOUBTFUL,
"BAD": BAD,
}
- _COL_LABELS: Dict[str, str] = {
- "flag": "quality_flag",
- "comment": "quality_comment",
- "cause": "quality_cause",
- }
def __init__(self):
super().__init__(forward=self._FORWARD)
@staticmethod
- def _getFieldFunctions(field: str, call_stack: MaterializedGraph) -> List[str]:
+ def _getFieldFunctions(field: str, call_stack: MaterializedGraph) -> List[SaQCFunction]:
"""
Return the names of all functions called on `field`
@@ -209,7 +210,7 @@ class DmpTranslator(Translator):
Note
----
- Could (and maybe should) be implemented as a method of `CalledStack`
+ Could (and maybe should) be implemented as a method of `CallGraph`
"""
return [f.name for l, f in call_stack if l.field == field]
--
GitLab
From 329c60e2dc66435690da419fe1d23f2dc2f0461d Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Sun, 25 Apr 2021 20:25:30 +0200
Subject: [PATCH 126/180] intermediate commit, waiting for GL#191
---
saqc/core/flags.py | 24 +++----
saqc/core/translator.py | 130 ++++++++++++++++++++++++----------
tests/core/test_translator.py | 44 ++++++++----
3 files changed, 133 insertions(+), 65 deletions(-)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index e98cc4170..3af83bfb9 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -10,6 +10,7 @@ from typing import Mapping, Union, Dict, DefaultDict, Optional, Type, Tuple, Ite
from saqc.constants import *
from saqc.core.history import History
+from saqc.lib.types import PandasLike
_VAL = Union[pd.Series, History]
DictLike = Union[
@@ -162,7 +163,6 @@ class Flags:
1 (255.0) (nan) (nan) 99.0
2 (-inf) (25.0) (0.0) 99.0
"""
-
def __init__(
self, raw_data: Optional[Union[DictLike, Flags]] = None, copy: bool = False
):
@@ -196,27 +196,25 @@ class Flags:
if k in result:
raise ValueError("raw_data must not have duplicate keys")
- # No, means no ! (copy)
- if isinstance(item, History) and not copy:
- result[k] = item
- continue
-
if isinstance(item, pd.Series):
item = item.to_frame(name=0)
elif isinstance(item, dios.DictOfSeries):
item = item.to_df()
- elif isinstance(item, (History, pd.DataFrame)):
- pass
+
+ if isinstance(item, pd.DataFrame):
+ item = History(
+ pd.DataFrame(UNFLAGGED, index=item.index, columns=[0], dtype=float),
+ ).append(History(item, copy=copy), force=True)
+
+ if isinstance(item, History):
+ if copy:
+ item = item.copy(deep=True)
else:
raise TypeError(
f"cannot init from '{type(data).__name__}' of '{type(item).__name__}'"
)
- hist = History(
- pd.DataFrame(UNFLAGGED, index=item.index, columns=[0], dtype=float),
- copy=copy
- )
- result[k] = hist.append(History(item, copy=copy), force=True)
+ result[k] = item
return result
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
index 60e171d00..237cf7a86 100644
--- a/saqc/core/translator.py
+++ b/saqc/core/translator.py
@@ -4,7 +4,9 @@
from __future__ import annotations
import json
-from typing import Dict, List, Optional, Union
+from saqc.core.history import History
+from saqc.core.lib import SaQCFunction, ColumnSelector
+from typing import Dict, List, Optional, Union, Any, Tuple
import numpy as np
import pandas as pd
@@ -12,8 +14,8 @@ import pandas as pd
from dios import DictOfSeries
from saqc.core.flags import Flags, UNTOUCHED, UNFLAGGED, GOOD, DOUBTFUL, BAD
-from saqc.core.history import History
from saqc.lib.types import ExternalFlag, MaterializedGraph
+from saqc.core.register import FUNC_MAP
# to_mask as part of th translator
@@ -124,7 +126,7 @@ class Translator:
return flag # type: ignore -> if flag is in `self._backward` it is of type float
return self._forward[flag]
- def forward(self, flags: pd.DataFrame) -> Flags:
+ def forward(self, flags: pd.DataFrame) -> Tuple[Flags, MaterializedGraph]:
"""
Translate from 'external flags' to 'internal flags'
@@ -137,9 +139,16 @@ class Translator:
-------
Flags object
"""
- return Flags(self._translate(flags, self._forward))
+ tflags = Flags(self._translate(flags, self._forward))
- def backward(self, flags: Flags, call_stack: MaterializedGraph) -> pd.DataFrame:
+ graph = []
+ for field in tflags.columns:
+ if (tflags[field] >= UNFLAGGED).any():
+ graph.append((ColumnSelector(field), SaQCFunction))
+
+ return tflags, graph
+
+ def backward(self, flags: Flags, call_graph: MaterializedGraph) -> pd.DataFrame:
"""
Translate from 'internal flags' to 'external flags'
@@ -156,7 +165,6 @@ class Translator:
-------
pd.DataFrame
"""
- # NOTE:
return self._translate(flags, self._backward).to_df()
@@ -212,7 +220,7 @@ class DmpTranslator(Translator):
----
Could (and maybe should) be implemented as a method of `CallGraph`
"""
- return [f.name for l, f in call_stack if l.field == field]
+ return [f for l, f in call_stack if l.field == field]
def forward(self, flags: pd.DataFrame) -> Flags:
"""
@@ -230,15 +238,51 @@ class DmpTranslator(Translator):
cols = flags.columns
if not isinstance(cols, pd.MultiIndex):
raise TypeError("DMP-Flags need mult-index columns")
- if set(cols.get_level_values(1)) != set(self._COL_LABELS.values()):
+ col_labels = {"quality_flag", "quality_comment", "qualty_cause"}
+ if set(cols.get_level_values(1)) != col_labels:
raise TypeError(
- f"DMP-Flags expect the labels 'list(self._COL_LABELS.values)' in the secondary level"
+ f"DMP-Flags expect the labels 'list(col_labes)' in the secondary level"
)
- qflags = flags.xs(key=self._COL_LABELS["flag"], axis="columns", level=1)
- return super().forward(qflags) # type: ignore
+ qflags = flags.xs(key="quality_flag", axis="columns", level=1)
- def backward(self, flags: Flags, call_stack: MaterializedGraph) -> pd.DataFrame:
+ graph: MaterializedGraph = []
+
+ for field in qflags.columns:
+ loc = ColumnSelector(field=field, target="field", regex=False)
+ func = self._toFunc(
+ comments=flags.loc[:, (field, "quality_comment")],
+ causes=flags.loc[:, (field, "quality_cause")],
+ )
+ graph.append((loc, func))
+
+ tflags = super().forward(qflags) # type: ignore
+ return tflags, graph
+
+
+ @staticmethod
+ def _toFunc(comments: pd.Series, causes: pd.Series) -> SaQCFunction:
+ data = pd.DataFrame(comments.apply(json.loads).to_list())
+ data["cause"] = causes
+ data = (data
+ .fillna("")
+ .loc[data["test"].astype(bool)]
+ .drop_duplicates())
+
+ if len(data) > 1:
+ raise ValueError("inconsistent flag data given")
+ elif len(data) == 1:
+ data = data.squeeze()
+ elif data.empty:
+ data = {"test": "", "cause": "", "comment": ""}
+
+ return SaQCFunction(
+ name=data["test"],
+ function=FUNC_MAP.get(data["test"], lambda x: x),
+ cause=data["cause"], comment=data["comment"]
+ )
+
+ def backward(self, flags: Flags, call_graph: MaterializedGraph) -> pd.DataFrame:
"""
Translate from 'internal flags' to 'external flags'
@@ -253,26 +297,29 @@ class DmpTranslator(Translator):
-------
pd.DataFrame
"""
- tflags = super().backward(flags, call_stack)
+ tflags = super().backward(flags, call_graph)
out = {}
for field in tflags.columns:
- flag_history = flags.history[field]
- flag_pos = flag_history.idxmax()
- flag_funcs = self._getFieldFunctions(field, call_stack)
- # NOTE:
- # we prepend empty strings to handle default columns in `Flags`
- # and potentially given flags not generated during the saqc run,
- # represented by `call_stack`
- flag_funcs = (
- [""] * (len(flag_history.hist.columns) - len(flag_funcs))
- ) + flag_funcs
- var_flags = {
- self._COL_LABELS["flag"]: tflags[field],
- self._COL_LABELS["comment"]: flag_pos.apply(
- lambda p: json.dumps({"test": flag_funcs[p]})
- ),
- self._COL_LABELS["cause"]: "",
- }
+ flag_call_history = self._getFieldFunctions(field, call_graph)
+ flag_pos = flags.history[field].idxmax() - 1 # to account for the default column
+ comments, causes = [], []
+ for p in flag_pos:
+ if p < 0:
+ comment = json.dumps({"test": "", "comment": ""})
+ cause = ""
+ else:
+ func = flag_call_history[p]
+ cause = func.keywords.get("cause", self.ARGUMENTS["cause"])
+ comment = json.dumps(
+ {
+ "test": getattr(func, "name", ""),
+ "comment": func.keywords.get("comment", self.ARGUMENTS["comment"]),
+ }
+ )
+ causes.append(cause)
+ comments.append(comment)
+
+ var_flags = {"quality_flag": tflags[field], "quality_comment": comments, "quality_cause": causes}
out[field] = pd.DataFrame(var_flags)
return pd.concat(out, axis="columns")
@@ -309,18 +356,22 @@ class PositionalTranslator(Translator):
Flags object
"""
data = {}
+ graph = []
for field in flags.columns:
# drop the first column (i.e. the '9')
fflags = pd.DataFrame(
- flags[field].astype(str).apply(tuple).tolist(), index=flags[field].index
- ).iloc[:, 1:]
-
- tflags = super().forward(fflags.astype(int)).toFrame()
- tflags.insert(
- loc=0, column=0, value=pd.Series(UNFLAGGED, index=fflags.index)
+ flags[field].astype(str).str.slice(start=1).apply(tuple).tolist(), index=flags[field].index
)
+
+ tflags, tgraph = super().forward(fflags.astype(int))
+ tflags = tflags.toFrame()
+ if tflags.empty:
+ tflags = pd.DataFrame(UNFLAGGED, index=fflags.index, columns=[0], dtype=float)
+
data[field] = tflags
- return Flags(data)
+ graph.extend(tgraph)
+ return Flags(data), graph
+
def backward(self, flags: Flags, call_stack: MaterializedGraph) -> pd.DataFrame:
"""
@@ -341,9 +392,12 @@ class PositionalTranslator(Translator):
out = {}
for field in flags.columns:
thist = flags.history[field].hist.replace(self._BACKWARD)
+ # Concatenate the single flag values. There are faster and more
+ # complicated approaches (see former `PositionalFlagger`), but
+ # this method shouldn't be called that often
tflags = (
thist.astype(int).astype(str).apply(lambda x: x.sum(), axis="columns")
)
- # NOTE: work around the default first column history columns (see GL#182)
+ # take care for the default columns
out[field] = "9" + tflags.str.slice(start=1)
return pd.DataFrame(out).fillna(-9999).astype(int)
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index ec164afa5..4d7dd25db 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -16,6 +16,7 @@ from saqc.constants import UNFLAGGED, BAD, DOUBTFUL
from saqc.core.translator import PositionalTranslator, Translator, DmpTranslator
from saqc.core.flags import Flags
from saqc.core.core import SaQC
+from saqc.core.lib import SaQCFunction
from tests.common import initData
@@ -78,7 +79,6 @@ def test_backwardTranslationFail():
def test_dmpTranslator():
Selector = namedtuple("Selector", ["field"])
- Function = namedtuple("Function", ["name"])
translator = DmpTranslator()
keys = np.array(tuple(translator._backward.keys()) * 50)
@@ -86,10 +86,17 @@ def test_dmpTranslator():
flags[:, "var1"] = BAD
flags[:, "var1"] = DOUBTFUL
flags[:, "var2"] = BAD
+
+ ident = lambda x: x
to_call = [
- (Selector("var1"), Function("flagFoo")),
- (Selector("var1"), Function("flagBar")),
- (Selector("var2"), Function("flagFoo")),
+ # the initial columns
+ (Selector("var1"), SaQCFunction("flagInit", ident)),
+ (Selector("var2"), SaQCFunction("flagInit", ident)),
+ (Selector("var3"), SaQCFunction("flagInit", ident, comment="initial flags")),
+
+ (Selector("var1"), SaQCFunction("flagFoo", ident)),
+ (Selector("var1"), SaQCFunction("flagBar", ident, comment="I did it")),
+ (Selector("var2"), SaQCFunction("flagFoo", ident)),
]
tflags = translator.backward(flags, to_call)
assert set(tflags.columns.get_level_values(1)) == {
@@ -99,16 +106,16 @@ def test_dmpTranslator():
}
assert (tflags.loc[:, ("var1", "quality_flag")] == "DOUBTFUL").all(axis=None)
- assert (tflags.loc[:, ("var1", "quality_comment")] == '{"test": "flagBar"}').all(
+ assert (tflags.loc[:, ("var1", "quality_comment")] == '{"test": "flagBar", "comment": "I did it"}').all(
axis=None
)
assert (tflags.loc[:, ("var2", "quality_flag")] == "BAD").all(axis=None)
- assert (tflags.loc[:, ("var2", "quality_comment")] == '{"test": "flagFoo"}').all(
+ assert (tflags.loc[:, ("var2", "quality_comment")] == '{"test": "flagFoo", "comment": ""}').all(
axis=None
)
- assert (tflags.loc[:, ("var3", "quality_comment")] == '{"test": ""}').all(axis=None)
+ # assert (tflags.loc[:, ("var3", "quality_comment")] == '{"test": "flagInit", "comment": "initial flags"}').all(axis=None)
def test_positionalTranslator():
@@ -136,13 +143,17 @@ def test_positionalTranslatorIntegration():
for field in flags.columns:
assert flags[field].astype(str).str.match("^9[012]*$").all()
- round_trip = translator.backward(translator.forward(flags), saqc._computed)
+
+ fff, ccc = translator.forward(flags)
+ import pdb; pdb.set_trace()
+ round_trip = translator.backward(*translator.forward(flags))
assert (flags.values == round_trip.values).all()
assert (flags.index == round_trip.index).all()
assert (flags.columns == round_trip.columns).all()
+@pytest.mark.skip()
def test_dmpTranslatorIntegration():
data = initData(3)
@@ -163,13 +174,16 @@ def test_dmpTranslatorIntegration():
assert qfunc.isin({"", "breaks.flagMissing", "outliers.flagRange"}).all(axis=None)
assert (qcause == "").all(axis=None)
- round_trip = translator.backward(translator.forward(flags), saqc._computed)
+ round_trip = translator.backward(*translator.forward(flags))
assert round_trip.xs("quality_flag", axis="columns", level=1).equals(qflags)
- assert (
- round_trip.xs("quality_comment", axis="columns", level=1)
- .applymap(lambda v: json.loads(v)["test"] == "")
- .all(axis=None)
- )
+
+ assert (round_trip
+ .xs("quality_comment", axis="columns", level=1)
+ .equals(flags.xs("quality_comment", axis="columns", level=1)))
+
+ assert (round_trip
+ .xs("quality_cause", axis="columns", level=1)
+ .equals(flags.xs("quality_cause", axis="columns", level=1)))
def _buildupSaQCObjects():
@@ -210,6 +224,7 @@ def test_translationPreservesFlags():
assert expected.equals(got)
+@pytest.mark.skip()
def test_positionalTranslationPreservesFlags():
saqc1, saqc2 = _buildupSaQCObjects()
@@ -225,6 +240,7 @@ def test_positionalTranslationPreservesFlags():
assert expected.equals(got)
+@pytest.mark.skip()
def test_dmpTranslationPreservesFlags():
saqc1, saqc2 = _buildupSaQCObjects()
--
GitLab
From fb2ad8eee78e3f4fb39e0c921ee7e06d7d151a4d Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Mon, 26 Apr 2021 17:57:28 +0200
Subject: [PATCH 127/180] fixed missing initial UNFLAGGED column on flags init
---
saqc/core/flags.py | 42 +++++++++++++++++++++---------------------
1 file changed, 21 insertions(+), 21 deletions(-)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index 3af83bfb9..ef61ffc59 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -193,28 +193,19 @@ class Flags:
for k, item in data.items():
+ if not isinstance(k, str):
+ raise ValueError("column names must be string")
if k in result:
raise ValueError("raw_data must not have duplicate keys")
-
- if isinstance(item, pd.Series):
- item = item.to_frame(name=0)
- elif isinstance(item, dios.DictOfSeries):
- item = item.to_df()
-
- if isinstance(item, pd.DataFrame):
- item = History(
- pd.DataFrame(UNFLAGGED, index=item.index, columns=[0], dtype=float),
- ).append(History(item, copy=copy), force=True)
-
if isinstance(item, History):
- if copy:
- item = item.copy(deep=True)
- else:
+ result[k] = History(item, copy=True) if copy else item
+ continue
+ if not isinstance(item, pd.Series):
raise TypeError(
f"cannot init from '{type(data).__name__}' of '{type(item).__name__}'"
)
- result[k] = item
+ result[k] = _simpleHist(item.index).append(item, force=True)
return result
@@ -336,7 +327,7 @@ class Flags:
return
if key not in self._data:
- self._data[key] = History()
+ self._data[key] = _simpleHist(value.index)
self._data[key].append(value, force=True)
self._cache.pop(key, None)
@@ -465,8 +456,8 @@ def initFlagsLike(
name : str, default None
Only respected if `reference` is of type ``pd.Series``.
The column name that is used for the Flags. If ``None``
- the name of the series itself is taken, if this is also
- `None`, a ValueError is raised.
+ the name of the series itself is taken, if it is unset,
+ a ValueError is raised.
Notes
-----
@@ -509,8 +500,17 @@ def initFlagsLike(
if not isinstance(item, (pd.Series, History)):
raise TypeError("items in reference must be of type pd.Series")
- item = pd.DataFrame(initial_value, index=item.index, columns=[0], dtype=float)
-
- result[k] = History(item)
+ result[k] = _simpleHist(item.index, initial_value)
return Flags(result)
+
+
+def _simpleHist(index, initial_value: float = UNFLAGGED) -> History:
+ """
+ Make a single columned History from an index and an initial value.
+
+ Notes
+ -----
+ For internal use only.
+ """
+ return History(pd.DataFrame(initial_value, index=index, columns=[0], dtype=float))
--
GitLab
From 8d6460822b9704df42ae14fd1e3a5fabefc68ce2 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Mon, 26 Apr 2021 18:06:16 +0200
Subject: [PATCH 128/180] minor change
---
saqc/core/flags.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index ef61ffc59..91bc740b6 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -198,7 +198,7 @@ class Flags:
if k in result:
raise ValueError("raw_data must not have duplicate keys")
if isinstance(item, History):
- result[k] = History(item, copy=True) if copy else item
+ result[k] = item.copy() if copy else item
continue
if not isinstance(item, pd.Series):
raise TypeError(
--
GitLab
From 1bd36283633442c5dfe403dfbaecb414c298abd1 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Mon, 26 Apr 2021 17:57:28 +0200
Subject: [PATCH 129/180] fixed missing initial UNFLAGGED column on flags init
---
saqc/core/flags.py | 36 +++++++++++++++++++++---------------
1 file changed, 21 insertions(+), 15 deletions(-)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index 34b2fed23..ecbf9c84f 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -191,24 +191,21 @@ class Flags:
for k, item in data.items():
+ if not isinstance(k, str):
+ raise ValueError("column names must be string")
if k in result:
raise ValueError("raw_data must not have duplicate keys")
- # No, means no ! (copy)
- if isinstance(item, History) and not copy:
- result[k] = item
+ if isinstance(item, History):
+ result[k] = History(item, copy=True) if copy else item
continue
- if isinstance(item, pd.Series):
- item = item.to_frame(name=0)
- elif isinstance(item, History):
- pass
- else:
+ if not isinstance(item, pd.Series):
raise TypeError(
f"cannot init from {type(data.__name__)} of {type(item.__name__)}"
)
- result[k] = History(item, copy=copy)
+ result[k] = _simpleHist(item.index).append(item, force=True)
return result
@@ -327,7 +324,7 @@ class Flags:
return
if key not in self._data:
- self._data[key] = History()
+ self._data[key] = _simpleHist(value.index)
self._data[key].append(value, force=True)
self._cache.pop(key, None)
@@ -456,8 +453,8 @@ def initFlagsLike(
name : str, default None
Only respected if `reference` is of type ``pd.Series``.
The column name that is used for the Flags. If ``None``
- the name of the series itself is taken, if this is also
- `None`, a ValueError is raised.
+ the name of the series itself is taken, if it is unset,
+ a ValueError is raised.
Notes
-----
@@ -502,8 +499,17 @@ def initFlagsLike(
if not isinstance(item, (pd.Series, History)):
raise TypeError("items in reference must be of type pd.Series")
- item = pd.DataFrame(initial_value, index=item.index, columns=[0], dtype=float)
-
- result[k] = History(item)
+ result[k] = _simpleHist(item.index, initial_value)
return Flags(result)
+
+
+def _simpleHist(index, initial_value: float = UNFLAGGED) -> History:
+ """
+ Make a single columned History from an index and an initial value.
+
+ Notes
+ -----
+ For internal use only.
+ """
+ return History(pd.DataFrame(initial_value, index=index, columns=[0], dtype=float))
--
GitLab
From a4c7491bffcc4fd0d9fffc0a4272de3d05111f0c Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Mon, 26 Apr 2021 18:06:16 +0200
Subject: [PATCH 130/180] minor change
---
saqc/core/flags.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index ecbf9c84f..a37adf279 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -197,7 +197,7 @@ class Flags:
raise ValueError("raw_data must not have duplicate keys")
if isinstance(item, History):
- result[k] = History(item, copy=True) if copy else item
+ result[k] = item.copy() if copy else item
continue
if not isinstance(item, pd.Series):
--
GitLab
From 440adc21c32fea1a54b4d540db9e8cd2f5d0f9b2 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 28 Apr 2021 14:31:06 +0200
Subject: [PATCH 131/180] hardened flags and history - added checks for
UNFLAGGED column - copy now is explicit and does not rely on constructor -
removed initial_value from initFlagsLike
---
saqc/core/flags.py | 51 ++++++++++++++++++++++++++++++++-----------
saqc/core/history.py | 8 ++++++-
saqc/core/register.py | 2 +-
3 files changed, 46 insertions(+), 15 deletions(-)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index a37adf279..b933b061e 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -43,6 +43,7 @@ class _HistAccess:
raise TypeError("Not a History")
History._validateHistWithMask(value.hist, value.mask)
+ self.obj._validateHistForFlags(value)
self.obj._data[key] = value
self.obj._cache.pop(key, None)
@@ -170,10 +171,12 @@ class Flags:
raw_data = {}
if isinstance(raw_data, Flags):
- raw_data = raw_data._data
+ if copy:
+ raw_data = raw_data.copy()
+ self._data = raw_data._data
- # with python 3.7 dicts are insertion-ordered by default
- self._data = self._initFromRaw(raw_data, copy)
+ else:
+ self._data = self._initFromRaw(raw_data, copy)
# this is a simple cache that reduce the calculation of the flags
# from the entire history of a flag column. The _cache is filled
@@ -196,19 +199,43 @@ class Flags:
if k in result:
raise ValueError("raw_data must not have duplicate keys")
+ # a passed History is not altered. So if the passed History
+ # does not fit for Flags, we fail hard.
if isinstance(item, History):
- result[k] = item.copy() if copy else item
+ self._validateHistForFlags(item, colname=k)
+ if copy:
+ item = item.copy()
+ result[k] = item
continue
if not isinstance(item, pd.Series):
raise TypeError(
- f"cannot init from {type(data.__name__)} of {type(item.__name__)}"
+ f"cannot init from {type(data).__name__} of {type(item).__name__}"
)
+ # make a UNFLAGGED-column and then append the actual item
result[k] = _simpleHist(item.index).append(item, force=True)
return result
+ @staticmethod
+ def _validateHistForFlags(history: History, colname=None):
+ if history.empty:
+ return history
+
+ errm = f"History "
+ if colname:
+ errm += f"of column {colname} "
+
+ if any(history.hist[0] != UNFLAGGED):
+ raise ValueError(errm + "missing an UNFLAGGED-column at first position")
+
+ # this ensures that the mask does not shadow UNFLAGGED with a NaN.
+ if history.max().hasnans:
+ raise ValueError(errm + "is not valid (result of max() contains NaNs)")
+
+ return history
+
@property
def _constructor(self) -> Type["Flags"]:
return type(self)
@@ -388,7 +415,9 @@ class Flags:
-------
copy of flags
"""
- return self._constructor(self, copy=deep)
+ new = self._constructor()
+ new._data = {c: h.copy() if deep else h for c, h in self._data.items()}
+ return new
def __copy__(self, deep=True):
return self.copy(deep=deep)
@@ -436,7 +465,6 @@ class Flags:
def initFlagsLike(
reference: Union[pd.Series, DictLike, Flags],
- initial_value: float = UNFLAGGED,
name: str = None,
) -> Flags:
"""
@@ -447,9 +475,6 @@ def initFlagsLike(
reference : pd.DataFrame, pd.Series, dios.DictOfSeries, dict of pd.Series
The reference structure to initialize for.
- initial_value : float, default 0
- value to initialize the columns with
-
name : str, default None
Only respected if `reference` is of type ``pd.Series``.
The column name that is used for the Flags. If ``None``
@@ -499,12 +524,12 @@ def initFlagsLike(
if not isinstance(item, (pd.Series, History)):
raise TypeError("items in reference must be of type pd.Series")
- result[k] = _simpleHist(item.index, initial_value)
+ result[k] = _simpleHist(item.index)
return Flags(result)
-def _simpleHist(index, initial_value: float = UNFLAGGED) -> History:
+def _simpleHist(index) -> History:
"""
Make a single columned History from an index and an initial value.
@@ -512,4 +537,4 @@ def _simpleHist(index, initial_value: float = UNFLAGGED) -> History:
-----
For internal use only.
"""
- return History(pd.DataFrame(initial_value, index=index, columns=[0], dtype=float))
+ return History(pd.DataFrame(UNFLAGGED, index=index, columns=[0], dtype=float))
diff --git a/saqc/core/history.py b/saqc/core/history.py
index 5b1e1f056..2eed6c5f4 100644
--- a/saqc/core/history.py
+++ b/saqc/core/history.py
@@ -350,7 +350,13 @@ class History:
copy : History
the copied FH
"""
- return self._constructor(hist=self, copy=deep)
+ new = self._constructor()
+ new.hist = self.hist
+ new.mask = self.mask
+ if deep:
+ new.hist = new.hist.copy()
+ new.mask = new.mask.copy()
+ return new
def reindex(self, index: pd.Index, fill_value_last: float = UNFLAGGED) -> History:
"""
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 7ebf177e2..8a76e2d80 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -283,7 +283,7 @@ def _prepareFlags(flags: Flags, masking) -> Flags:
if masking == "none":
return flags.copy()
- return initFlagsLike(flags, initial_value=UNTOUCHED)
+ return initFlagsLike(flags)
def _restoreFlags(flags: Flags, old_state: CallState):
--
GitLab
From b7e84d132e6cc00087f914960562a5a130aedfb5 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 29 Apr 2021 09:26:00 +0200
Subject: [PATCH 132/180] accpet more argument types
---
saqc/core/history.py | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/saqc/core/history.py b/saqc/core/history.py
index 5b1e1f056..a4885cfe4 100644
--- a/saqc/core/history.py
+++ b/saqc/core/history.py
@@ -170,7 +170,7 @@ class History:
return self
- def append(self, value: Union[pd.Series, History], force=False) -> History:
+ def append(self, value: Union[pd.Series, pd.DataFrame, History], force=False) -> History:
"""
Create a new FH column and insert given pd.Series to it.
@@ -210,11 +210,15 @@ class History:
if isinstance(value, History):
return self._appendHistory(value, force=force)
- value = self._validateValue(value)
- if len(self) > 0 and not value.index.equals(self.index):
- raise ValueError("Index does not match")
+ if isinstance(value, pd.Series):
+ value = value.to_frame()
+
+ for _, val in value.items():
+ val = self._validateValue(val)
+ if len(self) > 0 and not val.index.equals(self.index):
+ raise ValueError("Index does not match")
- self._insert(value, pos=len(self), force=force)
+ self._insert(val, pos=len(self), force=force)
return self
def _appendHistory(self, value: History, force: bool = False):
--
GitLab
From a14894f9754ab020280c873e2019071f3d6ac7f5 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 29 Apr 2021 09:26:36 +0200
Subject: [PATCH 133/180] usable defaults
---
saqc/core/lib.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/saqc/core/lib.py b/saqc/core/lib.py
index ad288b086..ebe011df1 100644
--- a/saqc/core/lib.py
+++ b/saqc/core/lib.py
@@ -2,10 +2,12 @@
# -*- coding: utf-8 -*-
from dataclasses import dataclass
+import functools
from typing import Optional, Any
from typing_extensions import Literal
+from saqc.funcs.flagtools import flagDummy
class ColumnSelector:
def __init__(self, field, target=None, regex=False):
@@ -36,7 +38,7 @@ class ConfigController(APIController):
class SaQCFunction:
- def __init__(self, name = "unknown", function=lambda x: x, *args, **keywords):
+ def __init__(self, name="dummy", function=flagDummy, *args, **keywords):
self.name = name
self.func = function
self.args = args
--
GitLab
From 69fd9a1d2ae40611990a34cb272d239278c2950a Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 29 Apr 2021 10:07:47 +0200
Subject: [PATCH 134/180] fixed cyclic import
---
saqc/core/lib.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/saqc/core/lib.py b/saqc/core/lib.py
index ebe011df1..854d9f05a 100644
--- a/saqc/core/lib.py
+++ b/saqc/core/lib.py
@@ -7,7 +7,6 @@ import functools
from typing import Optional, Any
from typing_extensions import Literal
-from saqc.funcs.flagtools import flagDummy
class ColumnSelector:
def __init__(self, field, target=None, regex=False):
@@ -37,8 +36,12 @@ class ConfigController(APIController):
)
+
class SaQCFunction:
- def __init__(self, name="dummy", function=flagDummy, *args, **keywords):
+
+
+ def __init__(self, name="dummy", function=lambda data, _, flags, **kwargs: (data, flags), *args, **keywords):
+
self.name = name
self.func = function
self.args = args
--
GitLab
From 8f0efa65d256b2b474e33272219d25216e662526 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 29 Apr 2021 10:09:01 +0200
Subject: [PATCH 135/180] fixed positional translator, now that we have a
stable init-column in History
---
saqc/core/flags.py | 1 +
saqc/core/history.py | 4 +-
saqc/core/lib.py | 11 ++-
saqc/core/translator.py | 137 ++++++++++++++++++++++++----------
tests/core/test_translator.py | 41 +++++-----
5 files changed, 131 insertions(+), 63 deletions(-)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index 91bc740b6..36c73c484 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -163,6 +163,7 @@ class Flags:
1 (255.0) (nan) (nan) 99.0
2 (-inf) (25.0) (0.0) 99.0
"""
+
def __init__(
self, raw_data: Optional[Union[DictLike, Flags]] = None, copy: bool = False
):
diff --git a/saqc/core/history.py b/saqc/core/history.py
index a4885cfe4..a46cb4df0 100644
--- a/saqc/core/history.py
+++ b/saqc/core/history.py
@@ -170,7 +170,9 @@ class History:
return self
- def append(self, value: Union[pd.Series, pd.DataFrame, History], force=False) -> History:
+ def append(
+ self, value: Union[pd.Series, pd.DataFrame, History], force=False
+ ) -> History:
"""
Create a new FH column and insert given pd.Series to it.
diff --git a/saqc/core/lib.py b/saqc/core/lib.py
index 854d9f05a..fb7b7fd63 100644
--- a/saqc/core/lib.py
+++ b/saqc/core/lib.py
@@ -36,11 +36,14 @@ class ConfigController(APIController):
)
-
class SaQCFunction:
-
-
- def __init__(self, name="dummy", function=lambda data, _, flags, **kwargs: (data, flags), *args, **keywords):
+ def __init__(
+ self,
+ name="dummy",
+ function=lambda data, _, flags, **kwargs: (data, flags),
+ *args,
+ **keywords,
+ ):
self.name = name
self.func = function
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
index 237cf7a86..5a7cf569c 100644
--- a/saqc/core/translator.py
+++ b/saqc/core/translator.py
@@ -4,7 +4,6 @@
from __future__ import annotations
import json
-from saqc.core.history import History
from saqc.core.lib import SaQCFunction, ColumnSelector
from typing import Dict, List, Optional, Union, Any, Tuple
@@ -13,11 +12,19 @@ import pandas as pd
from dios import DictOfSeries
-from saqc.core.flags import Flags, UNTOUCHED, UNFLAGGED, GOOD, DOUBTFUL, BAD
+from saqc.core.flags import (
+ Flags,
+ _simpleHist,
+ UNTOUCHED,
+ UNFLAGGED,
+ GOOD,
+ DOUBTFUL,
+ BAD,
+)
+from saqc.core.history import History
from saqc.lib.types import ExternalFlag, MaterializedGraph
from saqc.core.register import FUNC_MAP
-# to_mask as part of th translator
ForwardMap = Dict[ExternalFlag, float]
BackwardMap = Dict[float, ExternalFlag]
@@ -50,7 +57,7 @@ class Translator:
# (internal) threshold flag above which values will be masked
TO_MASK: Union[float, bool] = True
- # additional arguments and default values the translation scheme accpepts
+ # additional arguments and default values the translation scheme accepts
ARGUMENTS: Dict[str, Any] = {}
def __init__(self, forward: ForwardMap, backward: Optional[BackwardMap] = None):
@@ -81,7 +88,8 @@ class Translator:
@staticmethod
def _translate(
- flags: Union[Flags, pd.DataFrame], trans_map: Union[ForwardMap, BackwardMap]
+ flags: Union[Flags, pd.DataFrame, pd.Series],
+ trans_map: Union[ForwardMap, BackwardMap],
) -> DictOfSeries:
"""
Translate a given flag data structure to another according to the
@@ -96,6 +104,9 @@ class Translator:
-------
pd.DataFrame, Flags
"""
+ if isinstance(flags, pd.Series):
+ flags = flags.to_frame()
+
out = DictOfSeries()
expected = pd.Index(trans_map.values())
for field in flags.columns:
@@ -126,6 +137,47 @@ class Translator:
return flag # type: ignore -> if flag is in `self._backward` it is of type float
return self._forward[flag]
+ @staticmethod
+ def _buildCallGraph(flags: Flags) -> MaterializedGraph:
+ """
+ build a call graph from the `Flags` and their `History`
+
+ As we usually don't have enough information (i.e. SaQC
+ function name and all used parameters) we generate dummy
+ functions here. These dummy functions unconditionally set
+ the `field` to the provided flags.
+
+ The idea is, to spit out an `MaterializedGraph`, that can
+ be used in replays of the original `SaQC` run in gives the
+ same result for the same input data set.
+
+ Parameters
+ ----------
+ flags : flags to generate a call graph for
+ """
+ out = []
+ for flag_column in flags.columns:
+ for _, hist_column in flags.history[flag_column].hist.items():
+ # NOTE:
+ # Close over `flags_column` and `history_column`
+ # to immitate the original function application,
+ # that we cannot replicate directly because of
+ # lacking information.
+ # I am not entirely sure, if closing over
+ # `flag_column` is really necessary or if we
+ # even should close over `flags`
+ def mapFlags(data, field, flags, **kwargs):
+ flags[flag_column] = hist_column
+ return data, flags
+
+ out.append(
+ (
+ ColumnSelector(flag_column),
+ SaQCFunction(name="initFlags", func=mapFlags),
+ )
+ )
+ return out
+
def forward(self, flags: pd.DataFrame) -> Tuple[Flags, MaterializedGraph]:
"""
Translate from 'external flags' to 'internal flags'
@@ -140,12 +192,7 @@ class Translator:
Flags object
"""
tflags = Flags(self._translate(flags, self._forward))
-
- graph = []
- for field in tflags.columns:
- if (tflags[field] >= UNFLAGGED).any():
- graph.append((ColumnSelector(field), SaQCFunction))
-
+ graph = self._buildCallGraph(tflags)
return tflags, graph
def backward(self, flags: Flags, call_graph: MaterializedGraph) -> pd.DataFrame:
@@ -204,7 +251,9 @@ class DmpTranslator(Translator):
super().__init__(forward=self._FORWARD)
@staticmethod
- def _getFieldFunctions(field: str, call_stack: MaterializedGraph) -> List[SaQCFunction]:
+ def _getFieldFunctions(
+ field: str, call_stack: MaterializedGraph
+ ) -> List[SaQCFunction]:
"""
Return the names of all functions called on `field`
@@ -222,7 +271,7 @@ class DmpTranslator(Translator):
"""
return [f for l, f in call_stack if l.field == field]
- def forward(self, flags: pd.DataFrame) -> Flags:
+ def forward(self, flags: pd.DataFrame) -> Tuple[Flags, MaterializedGraph]:
"""
Translate from 'extrnal flags' to 'internal flags'
@@ -259,15 +308,11 @@ class DmpTranslator(Translator):
tflags = super().forward(qflags) # type: ignore
return tflags, graph
-
@staticmethod
def _toFunc(comments: pd.Series, causes: pd.Series) -> SaQCFunction:
data = pd.DataFrame(comments.apply(json.loads).to_list())
data["cause"] = causes
- data = (data
- .fillna("")
- .loc[data["test"].astype(bool)]
- .drop_duplicates())
+ data = data.fillna("").loc[data["test"].astype(bool)].drop_duplicates()
if len(data) > 1:
raise ValueError("inconsistent flag data given")
@@ -279,7 +324,8 @@ class DmpTranslator(Translator):
return SaQCFunction(
name=data["test"],
function=FUNC_MAP.get(data["test"], lambda x: x),
- cause=data["cause"], comment=data["comment"]
+ cause=data["cause"],
+ comment=data["comment"],
)
def backward(self, flags: Flags, call_graph: MaterializedGraph) -> pd.DataFrame:
@@ -301,7 +347,9 @@ class DmpTranslator(Translator):
out = {}
for field in tflags.columns:
flag_call_history = self._getFieldFunctions(field, call_graph)
- flag_pos = flags.history[field].idxmax() - 1 # to account for the default column
+ flag_pos = (
+ flags.history[field].idxmax() - 1
+ ) # to account for the default column
comments, causes = [], []
for p in flag_pos:
if p < 0:
@@ -313,13 +361,19 @@ class DmpTranslator(Translator):
comment = json.dumps(
{
"test": getattr(func, "name", ""),
- "comment": func.keywords.get("comment", self.ARGUMENTS["comment"]),
+ "comment": func.keywords.get(
+ "comment", self.ARGUMENTS["comment"]
+ ),
}
)
causes.append(cause)
comments.append(comment)
- var_flags = {"quality_flag": tflags[field], "quality_comment": comments, "quality_cause": causes}
+ var_flags = {
+ "quality_flag": tflags[field],
+ "quality_comment": comments,
+ "quality_cause": causes,
+ }
out[field] = pd.DataFrame(var_flags)
return pd.concat(out, axis="columns")
@@ -342,9 +396,9 @@ class PositionalTranslator(Translator):
def __init__(self):
super().__init__(forward=self._FORWARD, backward=self._BACKWARD)
- def forward(self, flags: pd.DataFrame) -> Flags:
+ def forward(self, flags: pd.DataFrame) -> Tuple[Flags, MaterializedGraph]:
"""
- Translate from 'extrnal flags' to 'internal flags'
+ Translate from 'external flags' to 'internal flags'
Parameters
----------
@@ -355,23 +409,25 @@ class PositionalTranslator(Translator):
-------
Flags object
"""
+
data = {}
- graph = []
- for field in flags.columns:
- # drop the first column (i.e. the '9')
- fflags = pd.DataFrame(
- flags[field].astype(str).str.slice(start=1).apply(tuple).tolist(), index=flags[field].index
- )
+ for field, field_flags in flags.items():
- tflags, tgraph = super().forward(fflags.astype(int))
- tflags = tflags.toFrame()
- if tflags.empty:
- tflags = pd.DataFrame(UNFLAGGED, index=fflags.index, columns=[0], dtype=float)
+ # explode the flags into sperate columns and drop the leading `9`
+ df = pd.DataFrame(
+ field_flags.astype(str).str.slice(start=1).apply(tuple).tolist(),
+ index=field_flags.index,
+ ).astype(int)
- data[field] = tflags
- graph.extend(tgraph)
- return Flags(data), graph
+ # the exploded values + the an initial column are the History of `field`
+ fflags = super()._translate(df, self._FORWARD)
+ field_history = _simpleHist(field_flags.index).append(fflags.to_df())
+ data[field] = field_history
+ tflags = Flags(data)
+ graph = self._buildCallGraph(tflags)
+
+ return tflags, graph
def backward(self, flags: Flags, call_stack: MaterializedGraph) -> pd.DataFrame:
"""
@@ -398,6 +454,9 @@ class PositionalTranslator(Translator):
tflags = (
thist.astype(int).astype(str).apply(lambda x: x.sum(), axis="columns")
)
- # take care for the default columns
- out[field] = "9" + tflags.str.slice(start=1)
+ out[field] = "9"
+ if not tflags.empty:
+ # take care for the default columns
+ out[field] += tflags.str.slice(start=1)
+
return pd.DataFrame(out).fillna(-9999).astype(int)
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index 4d7dd25db..b520a8b8a 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -93,7 +93,6 @@ def test_dmpTranslator():
(Selector("var1"), SaQCFunction("flagInit", ident)),
(Selector("var2"), SaQCFunction("flagInit", ident)),
(Selector("var3"), SaQCFunction("flagInit", ident, comment="initial flags")),
-
(Selector("var1"), SaQCFunction("flagFoo", ident)),
(Selector("var1"), SaQCFunction("flagBar", ident, comment="I did it")),
(Selector("var2"), SaQCFunction("flagFoo", ident)),
@@ -106,14 +105,16 @@ def test_dmpTranslator():
}
assert (tflags.loc[:, ("var1", "quality_flag")] == "DOUBTFUL").all(axis=None)
- assert (tflags.loc[:, ("var1", "quality_comment")] == '{"test": "flagBar", "comment": "I did it"}').all(
- axis=None
- )
+ assert (
+ tflags.loc[:, ("var1", "quality_comment")]
+ == '{"test": "flagBar", "comment": "I did it"}'
+ ).all(axis=None)
assert (tflags.loc[:, ("var2", "quality_flag")] == "BAD").all(axis=None)
- assert (tflags.loc[:, ("var2", "quality_comment")] == '{"test": "flagFoo", "comment": ""}').all(
- axis=None
- )
+ assert (
+ tflags.loc[:, ("var2", "quality_comment")]
+ == '{"test": "flagFoo", "comment": ""}'
+ ).all(axis=None)
# assert (tflags.loc[:, ("var3", "quality_comment")] == '{"test": "flagInit", "comment": "initial flags"}').all(axis=None)
@@ -126,9 +127,9 @@ def test_positionalTranslator():
flags[2::3, "var1"] = BAD
tflags = translator.backward(flags, None) # type: ignore
- assert (tflags["var2"].replace(-9999, np.nan).dropna() == 9).all(axis=None)
- assert (tflags["var1"].iloc[1::3] == 9210).all(axis=None)
- assert (tflags["var1"].iloc[2::3] == 9002).all(axis=None)
+ assert (tflags["var2"].replace(-9999, np.nan).dropna() == 90).all(axis=None)
+ assert (tflags["var1"].iloc[1::3] == 90210).all(axis=None)
+ assert (tflags["var1"].iloc[2::3] == 90002).all(axis=None)
def test_positionalTranslatorIntegration():
@@ -138,14 +139,16 @@ def test_positionalTranslatorIntegration():
translator = PositionalTranslator()
saqc = SaQC(data=data, translator=translator)
- saqc = saqc.breaks.flagMissing(col).outliers.flagRange(col, min=3, max=10)
+ saqc = saqc.breaks.flagMissing(col).outliers.flagRange(
+ col, min=3, max=10, flag=DOUBTFUL
+ )
data, flags = saqc.getResult()
for field in flags.columns:
assert flags[field].astype(str).str.match("^9[012]*$").all()
- fff, ccc = translator.forward(flags)
- import pdb; pdb.set_trace()
+ assert (flags[col].astype(str).str.len() == 3).all()
+
round_trip = translator.backward(*translator.forward(flags))
assert (flags.values == round_trip.values).all()
@@ -177,13 +180,13 @@ def test_dmpTranslatorIntegration():
round_trip = translator.backward(*translator.forward(flags))
assert round_trip.xs("quality_flag", axis="columns", level=1).equals(qflags)
- assert (round_trip
- .xs("quality_comment", axis="columns", level=1)
- .equals(flags.xs("quality_comment", axis="columns", level=1)))
+ assert round_trip.xs("quality_comment", axis="columns", level=1).equals(
+ flags.xs("quality_comment", axis="columns", level=1)
+ )
- assert (round_trip
- .xs("quality_cause", axis="columns", level=1)
- .equals(flags.xs("quality_cause", axis="columns", level=1)))
+ assert round_trip.xs("quality_cause", axis="columns", level=1).equals(
+ flags.xs("quality_cause", axis="columns", level=1)
+ )
def _buildupSaQCObjects():
--
GitLab
From 96a668d47c8545656d83a706f602c3c8f20ae449 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Mon, 3 May 2021 14:00:15 +0200
Subject: [PATCH 136/180] make translations runtrips reproducable
---
saqc/core/core.py | 8 +-
saqc/core/flags.py | 2 +-
saqc/core/lib.py | 3 +
saqc/core/translator.py | 141 ++++++++++++++++++----------------
tests/core/test_translator.py | 107 ++++++++++++++++++--------
5 files changed, 160 insertions(+), 101 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index bee735a0c..a4c097a31 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -140,10 +140,12 @@ class SaQC(FuncModules):
# with regular expressions, we can't just reuse the original execution
# plan to infer all translation related information.
self._planned: CallGraph = [] # will be filled by calls to `_wrap`
- self._computed: MaterializedGraph = [] # will be filled in `evaluate`
+ self._computed: MaterializedGraph = self._translator._buildCallGraph(
+ self._flags
+ ) # will be filled in `evaluate`
@staticmethod
- def _initFlags(data, flags: Optional[Flags]):
+ def _initFlags(data: DictOfSeries, flags: Optional[Flags]) -> Flags:
"""
Init the internal Flags-object.
@@ -156,7 +158,7 @@ class SaQC(FuncModules):
# add columns that are present in data but not in flags
for c in data.columns.difference(flags.columns):
- flags[c] = pd.Series(UNFLAGGED, index=data[c].index, dtype=float)
+ flags[c] = initFlagsLike(data[c])
return flags
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index 36c73c484..751c0ed48 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -27,7 +27,7 @@ SelectT = Union[
Tuple[pd.Index, _Field],
Tuple[slice, _Field],
]
-ValueT = Union[pd.Series, Iterable, float]
+ValueT = Union[pd.Series, "Flags", Iterable, float]
class _HistAccess:
diff --git a/saqc/core/lib.py b/saqc/core/lib.py
index fb7b7fd63..e487f5bc3 100644
--- a/saqc/core/lib.py
+++ b/saqc/core/lib.py
@@ -14,6 +14,9 @@ class ColumnSelector:
self.target = target or field
self.regex = regex
+ def __repr__(self):
+ return f"{self.__class__.__name__}({self.field})"
+
# TODO: this seems obsolete
@dataclass
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
index 5a7cf569c..3f084e93a 100644
--- a/saqc/core/translator.py
+++ b/saqc/core/translator.py
@@ -1,11 +1,13 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
+# TODO: remove `MaterializedGraph` from `Translator.forward`
+
from __future__ import annotations
import json
from saqc.core.lib import SaQCFunction, ColumnSelector
-from typing import Dict, List, Optional, Union, Any, Tuple
+from typing import Dict, List, Optional, Union, Any, Tuple, Callable
import numpy as np
import pandas as pd
@@ -137,6 +139,24 @@ class Translator:
return flag # type: ignore -> if flag is in `self._backward` it is of type float
return self._forward[flag]
+ @staticmethod
+ def _generateInitFunction(
+ flag_name: str, history: pd.Series
+ ) -> Callable[[DictOfSeries, str, Flags, Any], Tuple[DictOfSeries, Flags]]:
+ # NOTE:
+ # Close over `flags_column` and `history_column`
+ # to immitate the original function application,
+ # that we cannot replicate directly because of
+ # lacking information.
+ # I am not entirely sure, if closing over
+ # `flag_column` is really necessary or if we
+ # even should close over `flags`
+ def mapFlags(data: DictOfSeries, field: str, flags: Flags, **kwargs):
+ flags[history.index, flag_name] = history
+ return data, flags
+
+ return mapFlags
+
@staticmethod
def _buildCallGraph(flags: Flags) -> MaterializedGraph:
"""
@@ -156,24 +176,18 @@ class Translator:
flags : flags to generate a call graph for
"""
out = []
- for flag_column in flags.columns:
- for _, hist_column in flags.history[flag_column].hist.items():
- # NOTE:
- # Close over `flags_column` and `history_column`
- # to immitate the original function application,
- # that we cannot replicate directly because of
- # lacking information.
- # I am not entirely sure, if closing over
- # `flag_column` is really necessary or if we
- # even should close over `flags`
- def mapFlags(data, field, flags, **kwargs):
- flags[flag_column] = hist_column
- return data, flags
-
+ for flag_name in flags.columns:
+ # skip the default column
+ for _, hist_column in tuple(flags.history[flag_name].hist.items())[1:]:
out.append(
(
- ColumnSelector(flag_column),
- SaQCFunction(name="initFlags", func=mapFlags),
+ ColumnSelector(flag_name),
+ SaQCFunction(
+ name="initFlags",
+ function=Translator._generateInitFunction(
+ flag_name, hist_column
+ ),
+ ),
)
)
return out
@@ -269,7 +283,7 @@ class DmpTranslator(Translator):
----
Could (and maybe should) be implemented as a method of `CallGraph`
"""
- return [f for l, f in call_stack if l.field == field]
+ return [SaQCFunction(name="")] + [f for l, f in call_stack if l.field == field]
def forward(self, flags: pd.DataFrame) -> Tuple[Flags, MaterializedGraph]:
"""
@@ -287,46 +301,47 @@ class DmpTranslator(Translator):
cols = flags.columns
if not isinstance(cols, pd.MultiIndex):
raise TypeError("DMP-Flags need mult-index columns")
- col_labels = {"quality_flag", "quality_comment", "qualty_cause"}
+ col_labels = {"quality_flag", "quality_comment", "quality_cause"}
if set(cols.get_level_values(1)) != col_labels:
raise TypeError(
- f"DMP-Flags expect the labels 'list(col_labes)' in the secondary level"
+ f"DMP-Flags expect the labels '{list(col_labels)}' in the secondary level"
)
qflags = flags.xs(key="quality_flag", axis="columns", level=1)
+ # We want to build a call graph from the given flags and as the DMP flags
+ # contain the name of last function that set a certain flag, we want to
+ # leverage this information
graph: MaterializedGraph = []
for field in qflags.columns:
- loc = ColumnSelector(field=field, target="field", regex=False)
- func = self._toFunc(
- comments=flags.loc[:, (field, "quality_comment")],
- causes=flags.loc[:, (field, "quality_cause")],
+
+ # extract relevant information from the comments
+ data = pd.DataFrame(
+ flags.loc[:, (field, "quality_comment")].apply(json.loads).to_list(),
+ index=flags.index,
)
- graph.append((loc, func))
+ data["causes"] = flags.loc[:, (field, "quality_cause")]
- tflags = super().forward(qflags) # type: ignore
- return tflags, graph
+ loc = ColumnSelector(field=field, target="field", regex=False)
- @staticmethod
- def _toFunc(comments: pd.Series, causes: pd.Series) -> SaQCFunction:
- data = pd.DataFrame(comments.apply(json.loads).to_list())
- data["cause"] = causes
- data = data.fillna("").loc[data["test"].astype(bool)].drop_duplicates()
-
- if len(data) > 1:
- raise ValueError("inconsistent flag data given")
- elif len(data) == 1:
- data = data.squeeze()
- elif data.empty:
- data = {"test": "", "cause": "", "comment": ""}
-
- return SaQCFunction(
- name=data["test"],
- function=FUNC_MAP.get(data["test"], lambda x: x),
- cause=data["cause"],
- comment=data["comment"],
- )
+ # we can't infer information about the ordering of function calls,
+ # so we order the history by appearance
+ for _, group in data.fillna("").groupby(["test", "comment", "causes"]):
+
+ fname, comment, cause = group.iloc[0]
+ func = SaQCFunction(
+ name=fname,
+ function=Translator._generateInitFunction(
+ field, qflags.loc[group.index]
+ ),
+ comment=comment,
+ cause=cause,
+ )
+ graph.append((loc, func))
+
+ tflags, _ = super().forward(qflags)
+ return tflags, graph
def backward(self, flags: Flags, call_graph: MaterializedGraph) -> pd.DataFrame:
"""
@@ -344,28 +359,24 @@ class DmpTranslator(Translator):
pd.DataFrame
"""
tflags = super().backward(flags, call_graph)
+
out = {}
+ # import pdb; pdb.set_trace()
for field in tflags.columns:
- flag_call_history = self._getFieldFunctions(field, call_graph)
- flag_pos = (
- flags.history[field].idxmax() - 1
- ) # to account for the default column
+ flag_call_history = self._getFieldFunctions(field, call_graph)[1:]
+ flag_pos = flags.history[field].idxmax()
comments, causes = [], []
for p in flag_pos:
- if p < 0:
- comment = json.dumps({"test": "", "comment": ""})
- cause = ""
- else:
- func = flag_call_history[p]
- cause = func.keywords.get("cause", self.ARGUMENTS["cause"])
- comment = json.dumps(
- {
- "test": getattr(func, "name", ""),
- "comment": func.keywords.get(
- "comment", self.ARGUMENTS["comment"]
- ),
- }
- )
+ func = flag_call_history[p]
+ cause = func.keywords.get("cause", self.ARGUMENTS["cause"])
+ comment = json.dumps(
+ {
+ "test": func.name,
+ "comment": func.keywords.get(
+ "comment", self.ARGUMENTS["comment"]
+ ),
+ }
+ )
causes.append(cause)
comments.append(comment)
@@ -425,7 +436,7 @@ class PositionalTranslator(Translator):
data[field] = field_history
tflags = Flags(data)
- graph = self._buildCallGraph(tflags)
+ graph = Translator._buildCallGraph(tflags)
return tflags, graph
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index b520a8b8a..d47e395d6 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -2,7 +2,6 @@
# -*- coding: utf-8 -*-
import json
-from collections import namedtuple
from typing import Dict, Union, Sequence
import numpy as np
@@ -13,10 +12,16 @@ import pytest
from dios import DictOfSeries
from saqc.constants import UNFLAGGED, BAD, DOUBTFUL
-from saqc.core.translator import PositionalTranslator, Translator, DmpTranslator
+from saqc.core.translator import (
+ FloatTranslator,
+ PositionalTranslator,
+ Translator,
+ DmpTranslator,
+)
from saqc.core.flags import Flags
from saqc.core.core import SaQC
-from saqc.core.lib import SaQCFunction
+from saqc.core.lib import APIController, SaQCFunction, ColumnSelector
+from saqc.funcs.flagtools import flagDummy
from tests.common import initData
@@ -78,8 +83,6 @@ def test_backwardTranslationFail():
def test_dmpTranslator():
- Selector = namedtuple("Selector", ["field"])
-
translator = DmpTranslator()
keys = np.array(tuple(translator._backward.keys()) * 50)
flags = _genFlags({"var1": keys, "var2": keys, "var3": keys})
@@ -87,15 +90,20 @@ def test_dmpTranslator():
flags[:, "var1"] = DOUBTFUL
flags[:, "var2"] = BAD
- ident = lambda x: x
to_call = [
# the initial columns
- (Selector("var1"), SaQCFunction("flagInit", ident)),
- (Selector("var2"), SaQCFunction("flagInit", ident)),
- (Selector("var3"), SaQCFunction("flagInit", ident, comment="initial flags")),
- (Selector("var1"), SaQCFunction("flagFoo", ident)),
- (Selector("var1"), SaQCFunction("flagBar", ident, comment="I did it")),
- (Selector("var2"), SaQCFunction("flagFoo", ident)),
+ (ColumnSelector("var1"), SaQCFunction("flagInit", flagDummy)),
+ (ColumnSelector("var2"), SaQCFunction("flagInit", flagDummy)),
+ (
+ ColumnSelector("var3"),
+ SaQCFunction("flagInit", flagDummy, comment="initial flags"),
+ ),
+ (ColumnSelector("var1"), SaQCFunction("flagFoo", flagDummy)),
+ (
+ ColumnSelector("var1"),
+ SaQCFunction("flagBar", flagDummy, comment="I did it"),
+ ),
+ (ColumnSelector("var2"), SaQCFunction("flagFoo", flagDummy)),
]
tflags = translator.backward(flags, to_call)
assert set(tflags.columns.get_level_values(1)) == {
@@ -116,7 +124,10 @@ def test_dmpTranslator():
== '{"test": "flagFoo", "comment": ""}'
).all(axis=None)
- # assert (tflags.loc[:, ("var3", "quality_comment")] == '{"test": "flagInit", "comment": "initial flags"}').all(axis=None)
+ assert (
+ tflags.loc[flags["var3"] == BAD, ("var3", "quality_comment")]
+ == '{"test": "flagInit", "comment": "initial flags"}'
+ ).all(axis=None)
def test_positionalTranslator():
@@ -156,7 +167,6 @@ def test_positionalTranslatorIntegration():
assert (flags.columns == round_trip.columns).all()
-@pytest.mark.skip()
def test_dmpTranslatorIntegration():
data = initData(3)
@@ -192,8 +202,9 @@ def test_dmpTranslatorIntegration():
def _buildupSaQCObjects():
"""
- return two saqc object, whereas the flags from the previous run
- are reused
+ return two evaluated saqc objects calling the same functions,
+ whereas the flags from the evaluetion of the first objetc are
+ used as input flags of the second
"""
data = initData(3)
col = data.columns[0]
@@ -227,32 +238,64 @@ def test_translationPreservesFlags():
assert expected.equals(got)
-@pytest.mark.skip()
-def test_positionalTranslationPreservesFlags():
+def test_callHistoryYieldsSameResults():
+
+ # a simple SaQC run
+ data = initData(3)
+ col = data.columns[0]
+ saqc1 = SaQC(data=data)
+ saqc1 = saqc1.breaks.flagMissing(col, to_mask=False).outliers.flagRange(
+ col, min=3, max=10, to_mask=False
+ )
+ _, flags1 = saqc1.getResult(raw=True)
+
+ # generate a dummy call history from flags
+ translator = FloatTranslator()
+ graph = translator._buildCallGraph(flags1)
+ saqc2 = SaQC(data=data)
+
+ # convert the call history into an excution plan and inject into a blank SaQC object
+ saqc2._planned = [(s, APIController(plot=False), f) for s, f in graph]
+ # replay the functions
+ _, flags2 = saqc2.getResult()
+
+ assert flags2.equals(flags1.toFrame())
+
+def test_multicallsPreserveHistory():
saqc1, saqc2 = _buildupSaQCObjects()
- translator = PositionalTranslator()
_, flags1 = saqc1.getResult(raw=True)
_, flags2 = saqc2.getResult(raw=True)
- tflags1 = translator.backward(flags1, saqc1._computed).astype(str)
- tflags2 = translator.backward(flags2, saqc2._computed).astype(str)
- for k in flags2.columns:
- expected = tflags1[k].str.slice(start=1) * 2
- got = tflags2[k].str.slice(start=1)
- assert expected.equals(got)
+ # check, that the `History` is duplicated
+ for col in flags2.columns:
+ hist1 = flags1.history[col].hist.loc[:, 1:]
+ hist2 = flags2.history[col].hist.loc[:, 1:]
+
+ hist21 = hist2.iloc[:, : len(hist1.columns)]
+ hist22 = hist2.iloc[:, len(hist1.columns) :]
+ hist21.columns = hist1.columns
+ hist22.columns = hist1.columns
-@pytest.mark.skip()
-def test_dmpTranslationPreservesFlags():
+ assert hist1.equals(hist21)
+ assert hist1.equals(hist22)
+ assert hist21.equals(hist22)
+
+ assert len(saqc2._computed) == len(saqc1._computed) * 2
+
+
+def test_positionalMulitcallsPreserveState():
saqc1, saqc2 = _buildupSaQCObjects()
+ translator = PositionalTranslator()
_, flags1 = saqc1.getResult(raw=True)
_, flags2 = saqc2.getResult(raw=True)
+ tflags1 = translator.backward(flags1, saqc1._computed).astype(str)
+ tflags2 = translator.backward(flags2, saqc2._computed).astype(str)
- translator = DmpTranslator()
- tflags1 = translator.backward(flags1, saqc1._computed)
- tflags2 = translator.backward(flags2, saqc2._computed)
-
- assert tflags1.equals(tflags2)
+ for k in flags2.columns:
+ expected = tflags1[k].str.slice(start=1) * 2
+ got = tflags2[k].str.slice(start=1)
+ assert expected.equals(got)
--
GitLab
From 60dcc8039bb80d55001f37d11328934a9dea77de Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Mon, 3 May 2021 14:44:51 +0200
Subject: [PATCH 137/180] fixed inconsistent dmp translation graph build
---
saqc/core/flags.py | 8 +++++++-
saqc/core/translator.py | 6 +++---
tests/core/test_translator.py | 1 +
3 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index 25c0faf2f..15e5f6f93 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -185,6 +185,12 @@ class Flags:
# with __getitem__ and cleared on any write access to self_data.
# There are not to may write access possibilities here so we don't
# have to much trouble.
+ # NOTE:
+ # `time pytest tests/core tests/funcs tests/integration` yields
+ # identical runtimes without or without the cache, the cache however
+ # adds code complexity, through the additional dictionary and stuff
+ # like the `_HistAccess`. If tests on real-world datasets give
+ # similar results, we should get rid of it.
self._cache = {}
@staticmethod
@@ -205,7 +211,7 @@ class Flags:
# a passed History is not altered. So if the passed History
# does not fit for Flags, we fail hard.
if isinstance(item, History):
- self._validateHistForFlags(item, colname=k)
+ Flags._validateHistForFlags(item, colname=k)
if copy:
item = item.copy()
result[k] = item
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
index 3f084e93a..8db4f7f74 100644
--- a/saqc/core/translator.py
+++ b/saqc/core/translator.py
@@ -327,8 +327,8 @@ class DmpTranslator(Translator):
# we can't infer information about the ordering of function calls,
# so we order the history by appearance
- for _, group in data.fillna("").groupby(["test", "comment", "causes"]):
-
+ # for _, group in data.fillna("").groupby(["test", "comment", "causes"]):
+ for _, group in data.loc[data["test"].replace("", np.nan).notna()].groupby(["test", "comment", "causes"]):
fname, comment, cause = group.iloc[0]
func = SaQCFunction(
name=fname,
@@ -363,7 +363,7 @@ class DmpTranslator(Translator):
out = {}
# import pdb; pdb.set_trace()
for field in tflags.columns:
- flag_call_history = self._getFieldFunctions(field, call_graph)[1:]
+ flag_call_history = self._getFieldFunctions(field, call_graph)
flag_pos = flags.history[field].idxmax()
comments, causes = [], []
for p in flag_pos:
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index d47e395d6..67ba5134f 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -188,6 +188,7 @@ def test_dmpTranslatorIntegration():
assert (qcause == "").all(axis=None)
round_trip = translator.backward(*translator.forward(flags))
+
assert round_trip.xs("quality_flag", axis="columns", level=1).equals(qflags)
assert round_trip.xs("quality_comment", axis="columns", level=1).equals(
--
GitLab
From 4f492d52fa4bf2d02498294733322bea9fc22f5f Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Tue, 4 May 2021 09:34:39 +0200
Subject: [PATCH 138/180] moved translator code into new core module
---
saqc/core/__init__.py | 1 +
saqc/core/core.py | 2 +-
saqc/core/translator.py | 244 +------------------
saqc/core/translator/__init__.py | 5 +
saqc/core/translator/basetranslator.py | 240 ++++++++++++++++++
saqc/core/translator/dmptranslator.py | 165 +++++++++++++
saqc/core/translator/positionaltranslator.py | 102 ++++++++
7 files changed, 521 insertions(+), 238 deletions(-)
create mode 100644 saqc/core/translator/__init__.py
create mode 100644 saqc/core/translator/basetranslator.py
create mode 100644 saqc/core/translator/dmptranslator.py
create mode 100644 saqc/core/translator/positionaltranslator.py
diff --git a/saqc/core/__init__.py b/saqc/core/__init__.py
index 0b4aacf33..6f1838ff2 100644
--- a/saqc/core/__init__.py
+++ b/saqc/core/__init__.py
@@ -4,3 +4,4 @@
from saqc.core.register import register
from saqc.core.flags import Flags, initFlagsLike
from saqc.core.core import SaQC, logger
+from saqc.core.translator import FloatTranslator, DmpTranslator, PositionalTranslator
diff --git a/saqc/core/core.py b/saqc/core/core.py
index a4c097a31..6068ac651 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -23,7 +23,7 @@ from saqc.core.register import FUNC_MAP, SaQCFunction
from saqc.core.modules import FuncModules
from saqc.funcs.tools import copy
from saqc.lib.plotting import plotHook, plotAllHook
-from saqc.core.translator import FloatTranslator, Translator
+from saqc.core.translator.basetranslator import Translator, FloatTranslator
from saqc.lib.types import ExternalFlag, CallGraph, MaterializedGraph, PandasLike
from saqc.constants import BAD
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
index 8db4f7f74..05c73bc8b 100644
--- a/saqc/core/translator.py
+++ b/saqc/core/translator.py
@@ -1,31 +1,29 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-# TODO: remove `MaterializedGraph` from `Translator.forward`
-
+# TODO:
+# - remove `MaterializedGraph` from `Translator.forward`
+# - own directory
+# - parameter checking
+# - fix exe
from __future__ import annotations
-import json
-from saqc.core.lib import SaQCFunction, ColumnSelector
-from typing import Dict, List, Optional, Union, Any, Tuple, Callable
+from typing import Dict, Optional, Union, Any, Tuple, Callable
import numpy as np
import pandas as pd
from dios import DictOfSeries
+from saqc.core.lib import SaQCFunction, ColumnSelector
from saqc.core.flags import (
Flags,
- _simpleHist,
- UNTOUCHED,
UNFLAGGED,
GOOD,
DOUBTFUL,
BAD,
)
-from saqc.core.history import History
from saqc.lib.types import ExternalFlag, MaterializedGraph
-from saqc.core.register import FUNC_MAP
ForwardMap = Dict[ExternalFlag, float]
@@ -243,231 +241,3 @@ class FloatTranslator(Translator):
def __init__(self):
super().__init__(self._FORWARD)
-
-
-class DmpTranslator(Translator):
-
- """
- Implements the translation from and to the flagging scheme implemented in
- the UFZ - Datamanagementportal
- """
-
- ARGUMENTS = {"comment": "", "cause": ""}
-
- _FORWARD: Dict[str, float] = {
- "NIL": UNFLAGGED,
- "OK": GOOD,
- "DOUBTFUL": DOUBTFUL,
- "BAD": BAD,
- }
-
- def __init__(self):
- super().__init__(forward=self._FORWARD)
-
- @staticmethod
- def _getFieldFunctions(
- field: str, call_stack: MaterializedGraph
- ) -> List[SaQCFunction]:
- """
- Return the names of all functions called on `field`
-
- Parameters
- ----------
- field: str
- variable/column name
-
- call_stack : List
- The saqc functions called to generate the given `flags` (i.e. `SaQC._computed`)
-
- Note
- ----
- Could (and maybe should) be implemented as a method of `CallGraph`
- """
- return [SaQCFunction(name="")] + [f for l, f in call_stack if l.field == field]
-
- def forward(self, flags: pd.DataFrame) -> Tuple[Flags, MaterializedGraph]:
- """
- Translate from 'extrnal flags' to 'internal flags'
-
- Parameters
- ----------
- flags : pd.DataFrame
- The external flags to translate
-
- Returns
- -------
- Flags object
- """
- cols = flags.columns
- if not isinstance(cols, pd.MultiIndex):
- raise TypeError("DMP-Flags need mult-index columns")
- col_labels = {"quality_flag", "quality_comment", "quality_cause"}
- if set(cols.get_level_values(1)) != col_labels:
- raise TypeError(
- f"DMP-Flags expect the labels '{list(col_labels)}' in the secondary level"
- )
-
- qflags = flags.xs(key="quality_flag", axis="columns", level=1)
-
- # We want to build a call graph from the given flags and as the DMP flags
- # contain the name of last function that set a certain flag, we want to
- # leverage this information
- graph: MaterializedGraph = []
-
- for field in qflags.columns:
-
- # extract relevant information from the comments
- data = pd.DataFrame(
- flags.loc[:, (field, "quality_comment")].apply(json.loads).to_list(),
- index=flags.index,
- )
- data["causes"] = flags.loc[:, (field, "quality_cause")]
-
- loc = ColumnSelector(field=field, target="field", regex=False)
-
- # we can't infer information about the ordering of function calls,
- # so we order the history by appearance
- # for _, group in data.fillna("").groupby(["test", "comment", "causes"]):
- for _, group in data.loc[data["test"].replace("", np.nan).notna()].groupby(["test", "comment", "causes"]):
- fname, comment, cause = group.iloc[0]
- func = SaQCFunction(
- name=fname,
- function=Translator._generateInitFunction(
- field, qflags.loc[group.index]
- ),
- comment=comment,
- cause=cause,
- )
- graph.append((loc, func))
-
- tflags, _ = super().forward(qflags)
- return tflags, graph
-
- def backward(self, flags: Flags, call_graph: MaterializedGraph) -> pd.DataFrame:
- """
- Translate from 'internal flags' to 'external flags'
-
- Parameters
- ----------
- flags : pd.DataFrame
- The external flags to translate
- call_stack : List
- The saqc functions called to generate the given `flags` (i.e. `SaQC._computed`)
-
- Returns
- -------
- pd.DataFrame
- """
- tflags = super().backward(flags, call_graph)
-
- out = {}
- # import pdb; pdb.set_trace()
- for field in tflags.columns:
- flag_call_history = self._getFieldFunctions(field, call_graph)
- flag_pos = flags.history[field].idxmax()
- comments, causes = [], []
- for p in flag_pos:
- func = flag_call_history[p]
- cause = func.keywords.get("cause", self.ARGUMENTS["cause"])
- comment = json.dumps(
- {
- "test": func.name,
- "comment": func.keywords.get(
- "comment", self.ARGUMENTS["comment"]
- ),
- }
- )
- causes.append(cause)
- comments.append(comment)
-
- var_flags = {
- "quality_flag": tflags[field],
- "quality_comment": comments,
- "quality_cause": causes,
- }
- out[field] = pd.DataFrame(var_flags)
- return pd.concat(out, axis="columns")
-
-
-class PositionalTranslator(Translator):
-
- """
- Implements the translation from and to the flagging scheme implemented by CHS
- """
-
- _FORWARD: Dict[int, float] = {0: UNFLAGGED, 1: DOUBTFUL, 2: BAD}
- _BACKWARD: Dict[float, int] = {
- UNTOUCHED: 0,
- UNFLAGGED: 0,
- GOOD: 0,
- DOUBTFUL: 1,
- BAD: 2,
- }
-
- def __init__(self):
- super().__init__(forward=self._FORWARD, backward=self._BACKWARD)
-
- def forward(self, flags: pd.DataFrame) -> Tuple[Flags, MaterializedGraph]:
- """
- Translate from 'external flags' to 'internal flags'
-
- Parameters
- ----------
- flags : pd.DataFrame
- The external flags to translate
-
- Returns
- -------
- Flags object
- """
-
- data = {}
- for field, field_flags in flags.items():
-
- # explode the flags into sperate columns and drop the leading `9`
- df = pd.DataFrame(
- field_flags.astype(str).str.slice(start=1).apply(tuple).tolist(),
- index=field_flags.index,
- ).astype(int)
-
- # the exploded values + the an initial column are the History of `field`
- fflags = super()._translate(df, self._FORWARD)
- field_history = _simpleHist(field_flags.index).append(fflags.to_df())
- data[field] = field_history
-
- tflags = Flags(data)
- graph = Translator._buildCallGraph(tflags)
-
- return tflags, graph
-
- def backward(self, flags: Flags, call_stack: MaterializedGraph) -> pd.DataFrame:
- """
- Translate from 'internal flags' to 'external flags'
-
- Parameters
- ----------
- flags : pd.DataFrame
- The external flags to translate
- call_stack : List
- The saqc functions called to generate the given `flags` (i.e. `SaQC._computed`)
- `call_stack` is not evaluated here.
-
- Returns
- -------
- pd.DataFrame
- """
- out = {}
- for field in flags.columns:
- thist = flags.history[field].hist.replace(self._BACKWARD)
- # Concatenate the single flag values. There are faster and more
- # complicated approaches (see former `PositionalFlagger`), but
- # this method shouldn't be called that often
- tflags = (
- thist.astype(int).astype(str).apply(lambda x: x.sum(), axis="columns")
- )
- out[field] = "9"
- if not tflags.empty:
- # take care for the default columns
- out[field] += tflags.str.slice(start=1)
-
- return pd.DataFrame(out).fillna(-9999).astype(int)
diff --git a/saqc/core/translator/__init__.py b/saqc/core/translator/__init__.py
new file mode 100644
index 000000000..97c92d924
--- /dev/null
+++ b/saqc/core/translator/__init__.py
@@ -0,0 +1,5 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+from saqc.core.translator.basetranslator import Translator, FloatTranslator
+from saqc.core.translator.positionaltranslator import PositionalTranslator
+from saqc.core.translator.dmptranslator import DmpTranslator
diff --git a/saqc/core/translator/basetranslator.py b/saqc/core/translator/basetranslator.py
new file mode 100644
index 000000000..487b22f17
--- /dev/null
+++ b/saqc/core/translator/basetranslator.py
@@ -0,0 +1,240 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# TODO:
+# - remove `MaterializedGraph` from `Translator.forward`
+# - fix exe
+
+from __future__ import annotations
+
+from saqc.core.lib import SaQCFunction, ColumnSelector
+from typing import Dict, Optional, Union, Any, Tuple, Callable
+
+import numpy as np
+import pandas as pd
+
+from dios import DictOfSeries
+
+from saqc.core.flags import (
+ Flags,
+ UNFLAGGED,
+ BAD,
+)
+from saqc.lib.types import ExternalFlag, MaterializedGraph
+
+
+ForwardMap = Dict[ExternalFlag, float]
+BackwardMap = Dict[float, ExternalFlag]
+
+
+class Translator:
+ """
+ This class provides the basic translation mechanism and should serve as
+ a base class for every other translation scheme.
+
+ The general translation is realized through dictionary lookups, altough
+ we might need to extend this logic to also allow calls to translation
+ functions in the future. Currently at least one `dict` defining the
+ 'forward' translation from 'user flags' -> 'internal flags' needs to be
+ provided.
+ Optionally a second `dict` can be passed to map 'internal flags' -> 'user flags',
+ if the latter is not given, this 'backwards' translation will inferred as
+ the inverse of the 'forward' translation.
+
+ The translation mechanism imposes a few restrictions:
+ - The scheme must be well definied, i.e. we need a backward translation for
+ every forward translation (each value in `self._forward` needs a key in
+ `self._backward`).
+ - We need translations for the special flags `saqc.constants.UNFLAGGED` and
+ `saqc.constants.BAD`. That implies, that every valid translation scheme
+ provides at least one user flag that maps to `BAD` and one that maps to
+ `UNFLAGGED`.
+ """
+
+ # (internal) threshold flag above which values will be masked
+ TO_MASK: Union[float, bool] = True
+
+ # additional arguments and default values the translation scheme accepts
+ ARGUMENTS: Dict[str, Any] = {}
+
+ def __init__(self, forward: ForwardMap, backward: Optional[BackwardMap] = None):
+ """
+ Parameters
+ ----------
+ forward : dict
+ A mapping defining the forward translation of scalar flag values
+
+ backward : dict, optinal
+ A mapping defining the backward translation of scalar flag values.
+ If not given, `backward` is inferred from `forward`
+
+ Note
+ ----
+ `backward` needs to provide a mappinf for the two special flags
+ `saqc.core.UNFLAGGED`, `saqc.core.BAD`
+ """
+ # NOTE: we also add the keys to also allow the usage of internal flags
+ self._forward = forward
+ if backward is None:
+ backward = {v: k for k, v in forward.items()}
+ if {UNFLAGGED, BAD} - set(backward.keys()):
+ raise ValueError(
+ f"need translations for the special flags `UNFLAGGED` ({UNFLAGGED}) and `BAD` ({BAD})"
+ )
+ self._backward = backward
+
+ @staticmethod
+ def _translate(
+ flags: Union[Flags, pd.DataFrame, pd.Series],
+ trans_map: Union[ForwardMap, BackwardMap],
+ ) -> DictOfSeries:
+ """
+ Translate a given flag data structure to another according to the
+ mapping given in `trans_map`
+
+ Parameters
+ ----------
+ flags : Flags, pd.DataFrame
+ The flags to translate
+
+ Returns
+ -------
+ pd.DataFrame, Flags
+ """
+ if isinstance(flags, pd.Series):
+ flags = flags.to_frame()
+
+ out = DictOfSeries()
+ expected = pd.Index(trans_map.values())
+ for field in flags.columns:
+ out[field] = flags[field].replace(trans_map)
+ diff = pd.Index(out[field]).difference(expected)
+ if not diff.empty:
+ raise ValueError(
+ f"flags were not translated: {diff.drop_duplicates().to_list()}"
+ )
+ return out
+
+ def __call__(self, flag: ExternalFlag) -> float:
+ """
+ Translate a scalar 'external flag' to an 'internal flag'
+
+ Parameters
+ ----------
+ flag : float, int, str
+ The external flag to translate
+
+ Returns
+ -------
+ float
+ """
+ if flag not in self._forward:
+ if flag not in self._backward:
+ raise ValueError(f"invalid flag: {flag}")
+ return flag # type: ignore -> if flag is in `self._backward` it is of type float
+ return self._forward[flag]
+
+ @staticmethod
+ def _generateInitFunction(
+ flag_name: str, history: pd.Series
+ ) -> Callable[[DictOfSeries, str, Flags, Any], Tuple[DictOfSeries, Flags]]:
+ # NOTE:
+ # Close over `flags_column` and `history_column`
+ # to immitate the original function application,
+ # that we cannot replicate directly because of
+ # lacking information.
+ # I am not entirely sure, if closing over
+ # `flag_column` is really necessary or if we
+ # even should close over `flags`
+ def mapFlags(data: DictOfSeries, field: str, flags: Flags, **kwargs):
+ flags[history.index, flag_name] = history
+ return data, flags
+
+ return mapFlags
+
+ @staticmethod
+ def _buildCallGraph(flags: Flags) -> MaterializedGraph:
+ """
+ build a call graph from the `Flags` and their `History`
+
+ As we usually don't have enough information (i.e. SaQC
+ function name and all used parameters) we generate dummy
+ functions here. These dummy functions unconditionally set
+ the `field` to the provided flags.
+
+ The idea is, to spit out an `MaterializedGraph`, that can
+ be used in replays of the original `SaQC` run in gives the
+ same result for the same input data set.
+
+ Parameters
+ ----------
+ flags : flags to generate a call graph for
+ """
+ out = []
+ for flag_name in flags.columns:
+ # skip the default column
+ for _, hist_column in tuple(flags.history[flag_name].hist.items())[1:]:
+ out.append(
+ (
+ ColumnSelector(flag_name),
+ SaQCFunction(
+ name="initFlags",
+ function=Translator._generateInitFunction(
+ flag_name, hist_column
+ ),
+ ),
+ )
+ )
+ return out
+
+ def forward(self, flags: pd.DataFrame) -> Tuple[Flags, MaterializedGraph]:
+ """
+ Translate from 'external flags' to 'internal flags'
+
+ Parameters
+ ----------
+ flags : pd.DataFrame
+ The external flags to translate
+
+ Returns
+ -------
+ Flags object
+ """
+ tflags = Flags(self._translate(flags, self._forward))
+ graph = self._buildCallGraph(tflags)
+ return tflags, graph
+
+ def backward(self, flags: Flags, call_graph: MaterializedGraph) -> pd.DataFrame:
+ """
+ Translate from 'internal flags' to 'external flags'
+
+ Parameters
+ ----------
+ flags : pd.DataFrame
+ The external flags to translate
+ call_stack : List
+ The saqc functions called to generate the given `flags` (i.e. `SaQC._computed`)
+ `call_stack` is not evaluated here, it's presence only ensures, that subclasses
+ have access to it.
+
+ Returns
+ -------
+ pd.DataFrame
+ """
+ return self._translate(flags, self._backward).to_df()
+
+
+class FloatTranslator(Translator):
+
+ """
+ Acts as the default Translator, provides a changeable subset of the
+ internal float flags
+ """
+
+ _FORWARD: Dict[float, float] = {
+ -np.inf: -np.inf,
+ **{k: k for k in np.arange(0, 256, dtype=float)},
+ }
+
+ def __init__(self):
+ super().__init__(self._FORWARD)
diff --git a/saqc/core/translator/dmptranslator.py b/saqc/core/translator/dmptranslator.py
new file mode 100644
index 000000000..1f173b691
--- /dev/null
+++ b/saqc/core/translator/dmptranslator.py
@@ -0,0 +1,165 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import json
+from typing import List, Tuple
+
+import numpy as np
+import pandas as pd
+
+from saqc.core.lib import SaQCFunction, ColumnSelector
+from saqc.core.flags import (
+ Flags,
+ UNFLAGGED,
+ GOOD,
+ DOUBTFUL,
+ BAD,
+)
+from saqc.lib.types import MaterializedGraph
+from saqc.core.translator.basetranslator import Translator, ForwardMap
+
+
+class DmpTranslator(Translator):
+
+ """
+ Implements the translation from and to the flagging scheme implemented in
+ the UFZ - Datamanagementportal
+ """
+
+ ARGUMENTS = {"comment": "", "cause": ""}
+
+ _FORWARD: ForwardMap = {
+ "NIL": UNFLAGGED,
+ "OK": GOOD,
+ "DOUBTFUL": DOUBTFUL,
+ "BAD": BAD,
+ }
+
+ def __init__(self):
+ super().__init__(forward=self._FORWARD)
+
+ @staticmethod
+ def _getFieldFunctions(
+ field: str, call_stack: MaterializedGraph
+ ) -> List[SaQCFunction]:
+ """
+ Return the names of all functions called on `field`
+
+ Parameters
+ ----------
+ field: str
+ variable/column name
+
+ call_stack : List
+ The saqc functions called to generate the given `flags` (i.e. `SaQC._computed`)
+
+ Note
+ ----
+ Could (and maybe should) be implemented as a method of `CallGraph`
+ """
+ return [SaQCFunction(name="")] + [f for l, f in call_stack if l.field == field]
+
+ def forward(self, flags: pd.DataFrame) -> Tuple[Flags, MaterializedGraph]:
+ """
+ Translate from 'extrnal flags' to 'internal flags'
+
+ Parameters
+ ----------
+ flags : pd.DataFrame
+ The external flags to translate
+
+ Returns
+ -------
+ Flags object
+ """
+ cols = flags.columns
+ if not isinstance(cols, pd.MultiIndex):
+ raise TypeError("DMP-Flags need mult-index columns")
+ col_labels = {"quality_flag", "quality_comment", "quality_cause"}
+ if set(cols.get_level_values(1)) != col_labels:
+ raise TypeError(
+ f"DMP-Flags expect the labels '{list(col_labels)}' in the secondary level"
+ )
+
+ qflags = flags.xs(key="quality_flag", axis="columns", level=1)
+
+ # We want to build a call graph from the given flags and as the DMP flags
+ # contain the name of last function that set a certain flag, we want to
+ # leverage this information
+ graph: MaterializedGraph = []
+
+ for field in qflags.columns:
+
+ # extract relevant information from the comments
+ data = pd.DataFrame(
+ flags.loc[:, (field, "quality_comment")].apply(json.loads).to_list(),
+ index=flags.index,
+ )
+ data["causes"] = flags.loc[:, (field, "quality_cause")]
+
+ loc = ColumnSelector(field=field, target="field", regex=False)
+
+ # we can't infer information about the ordering of function calls,
+ # so we order the history by appearance
+ # for _, group in data.fillna("").groupby(["test", "comment", "causes"]):
+ for _, group in data.loc[data["test"].replace("", np.nan).notna()].groupby(["test", "comment", "causes"]):
+ fname, comment, cause = group.iloc[0]
+ func = SaQCFunction(
+ name=fname,
+ function=Translator._generateInitFunction(
+ field, qflags.loc[group.index]
+ ),
+ comment=comment,
+ cause=cause,
+ )
+ graph.append((loc, func))
+
+ tflags, _ = super().forward(qflags)
+ return tflags, graph
+
+ def backward(self, flags: Flags, call_graph: MaterializedGraph) -> pd.DataFrame:
+ """
+ Translate from 'internal flags' to 'external flags'
+
+ Parameters
+ ----------
+ flags : pd.DataFrame
+ The external flags to translate
+ call_stack : List
+ The saqc functions called to generate the given `flags` (i.e. `SaQC._computed`)
+
+ Returns
+ -------
+ pd.DataFrame
+ """
+ tflags = super().backward(flags, call_graph)
+
+ out = {}
+ # import pdb; pdb.set_trace()
+ for field in tflags.columns:
+ flag_call_history = self._getFieldFunctions(field, call_graph)
+ flag_pos = flags.history[field].idxmax()
+ comments, causes = [], []
+ for p in flag_pos:
+ func = flag_call_history[p]
+ cause = func.keywords.get("cause", self.ARGUMENTS["cause"])
+ comment = json.dumps(
+ {
+ "test": func.name,
+ "comment": func.keywords.get(
+ "comment", self.ARGUMENTS["comment"]
+ ),
+ }
+ )
+ causes.append(cause)
+ comments.append(comment)
+
+ var_flags = {
+ "quality_flag": tflags[field],
+ "quality_comment": comments,
+ "quality_cause": causes,
+ }
+ out[field] = pd.DataFrame(var_flags)
+ return pd.concat(out, axis="columns")
diff --git a/saqc/core/translator/positionaltranslator.py b/saqc/core/translator/positionaltranslator.py
new file mode 100644
index 000000000..dae62477b
--- /dev/null
+++ b/saqc/core/translator/positionaltranslator.py
@@ -0,0 +1,102 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+from __future__ import annotations
+
+from typing import Tuple
+
+import pandas as pd
+
+from saqc.core.flags import (
+ Flags,
+ _simpleHist,
+ UNTOUCHED,
+ UNFLAGGED,
+ GOOD,
+ DOUBTFUL,
+ BAD,
+)
+from saqc.lib.types import MaterializedGraph
+from saqc.core.translator.basetranslator import Translator, ForwardMap
+
+class PositionalTranslator(Translator):
+
+ """
+ Implements the translation from and to the flagging scheme implemented by CHS
+ """
+
+ _FORWARD: ForwardMap = {0: UNFLAGGED, 1: DOUBTFUL, 2: BAD}
+ _BACKWARD: BackwardMap = {
+ UNTOUCHED: 0,
+ UNFLAGGED: 0,
+ GOOD: 0,
+ DOUBTFUL: 1,
+ BAD: 2,
+ }
+
+ def __init__(self):
+ super().__init__(forward=self._FORWARD, backward=self._BACKWARD)
+
+ def forward(self, flags: pd.DataFrame) -> Tuple[Flags, MaterializedGraph]:
+ """
+ Translate from 'external flags' to 'internal flags'
+
+ Parameters
+ ----------
+ flags : pd.DataFrame
+ The external flags to translate
+
+ Returns
+ -------
+ Flags object
+ """
+
+ data = {}
+ for field, field_flags in flags.items():
+
+ # explode the flags into sperate columns and drop the leading `9`
+ df = pd.DataFrame(
+ field_flags.astype(str).str.slice(start=1).apply(tuple).tolist(),
+ index=field_flags.index,
+ ).astype(int)
+
+ # the exploded values + the an initial column are the History of `field`
+ fflags = super()._translate(df, self._FORWARD)
+ field_history = _simpleHist(field_flags.index).append(fflags.to_df())
+ data[field] = field_history
+
+ tflags = Flags(data)
+ graph = Translator._buildCallGraph(tflags)
+
+ return tflags, graph
+
+ def backward(self, flags: Flags, call_stack: MaterializedGraph) -> pd.DataFrame:
+ """
+ Translate from 'internal flags' to 'external flags'
+
+ Parameters
+ ----------
+ flags : pd.DataFrame
+ The external flags to translate
+ call_stack : List
+ The saqc functions called to generate the given `flags` (i.e. `SaQC._computed`)
+ `call_stack` is not evaluated here.
+
+ Returns
+ -------
+ pd.DataFrame
+ """
+ out = {}
+ for field in flags.columns:
+ thist = flags.history[field].hist.replace(self._BACKWARD)
+ # Concatenate the single flag values. There are faster and more
+ # complicated approaches (see former `PositionalFlagger`), but
+ # this method shouldn't be called that often
+ tflags = (
+ thist.astype(int).astype(str).apply(lambda x: x.sum(), axis="columns")
+ )
+ out[field] = "9"
+ if not tflags.empty:
+ # take care for the default columns
+ out[field] += tflags.str.slice(start=1)
+
+ return pd.DataFrame(out).fillna(-9999).astype(int)
--
GitLab
From 4452c3c4644a5b1510fd45a7bed5290a46b90ece Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Tue, 4 May 2021 10:04:39 +0200
Subject: [PATCH 139/180] exclude translation arguments from function keyword
checks
---
saqc/core/core.py | 6 +++---
saqc/core/translator/basetranslator.py | 2 +-
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 6068ac651..bc31aa81f 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -241,6 +241,7 @@ class SaQC(FuncModules):
data_result, flags_result = _saqcCallFunc(
sel, control, func, data, flags
)
+ _warnForUnusedKwargs(function, self._translator)
computed.append((sel, func))
except Exception as e:
_handleErrors(e, sel.field, control, func, self._error_policy)
@@ -356,12 +357,11 @@ def _saqcCallFunc(locator, controller, function, data, flags):
# we check the passed function-kwargs after the actual call, because now "hard" errors would already have been
# raised (Eg. `TypeError: got multiple values for argument 'data'`, when the user pass data=...)
- _warnForUnusedKwargs(function)
return data_result, flags_result
-def _warnForUnusedKwargs(func):
+def _warnForUnusedKwargs(func, translator: Translator):
"""Warn for unused kwargs, passed to a SaQC.function.
Parameters
@@ -388,7 +388,7 @@ def _warnForUnusedKwargs(func):
# there is no need to check for
# `kw in [KEYWORD_ONLY, VAR_KEYWORD or POSITIONAL_OR_KEYWORD]`
# because this would have raised an error beforehand.
- if kw not in sig_kws and kw not in ignore:
+ if kw not in sig_kws and kw not in ignore and kw not in translator.ARGUMENTS:
missing.append(kw)
if missing:
diff --git a/saqc/core/translator/basetranslator.py b/saqc/core/translator/basetranslator.py
index 487b22f17..32b8d55cf 100644
--- a/saqc/core/translator/basetranslator.py
+++ b/saqc/core/translator/basetranslator.py
@@ -231,7 +231,7 @@ class FloatTranslator(Translator):
internal float flags
"""
- _FORWARD: Dict[float, float] = {
+ _FORWARD: ForwardMap = {
-np.inf: -np.inf,
**{k: k for k in np.arange(0, 256, dtype=float)},
}
--
GitLab
From 74326e3e3c46642e999cf0844356d3d556fc58d5 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Tue, 4 May 2021 12:31:40 +0200
Subject: [PATCH 140/180] make translations available through the EXE
---
saqc/__init__.py | 2 +-
saqc/__main__.py | 39 +++++++++++++--------------
saqc/core/translator/dmptranslator.py | 6 ++---
tests/integration/test_integration.py | 25 ++++++++---------
4 files changed, 35 insertions(+), 37 deletions(-)
diff --git a/saqc/__init__.py b/saqc/__init__.py
index 6262ae74d..7fa49e8dd 100644
--- a/saqc/__init__.py
+++ b/saqc/__init__.py
@@ -5,4 +5,4 @@ __version__ = "1.4"
# import order: from small to big
from saqc.constants import *
-from saqc.core import register, initFlagsLike, Flags, SaQC
+from saqc.core import register, initFlagsLike, Flags, SaQC, FloatTranslator, DmpTranslator, PositionalTranslator
diff --git a/saqc/__main__.py b/saqc/__main__.py
index 1cafc99ed..50dab15cc 100644
--- a/saqc/__main__.py
+++ b/saqc/__main__.py
@@ -2,7 +2,6 @@
# -*- coding: utf-8 -*-
import logging
-import warnings
from functools import partial
from pathlib import Path
@@ -12,18 +11,16 @@ import numpy as np
import pandas as pd
import pyarrow as pa
-from saqc.constants import *
-from saqc.core import SaQC
+from saqc.core import SaQC, FloatTranslator, DmpTranslator, PositionalTranslator
logger = logging.getLogger("SaQC")
SCHEMES = {
- None: None,
- "numeric": NotImplemented,
- "category": NotImplemented,
- "dmp": NotImplemented,
+ "float": FloatTranslator,
+ "positional": PositionalTranslator,
+ "dmp": DmpTranslator,
}
@@ -89,7 +86,7 @@ def writeData(writer_dict, df, fname):
"-o", "--outfile", type=click.Path(exists=False), help="path to the output file"
)
@click.option(
- "--flagger",
+ "--scheme",
default=None,
type=click.Choice(SCHEMES.keys()),
help="the flagging scheme to use",
@@ -104,10 +101,7 @@ def writeData(writer_dict, df, fname):
@click.option(
"--fail/--no-fail", default=True, help="whether to stop the program run on errors"
)
-def main(config, data, flagger, outfile, nodata, log_level, fail):
-
- if SCHEMES[flagger] is NotImplemented:
- warnings.warn("--flagger is deprecated", DeprecationWarning)
+def main(config, data, scheme, outfile, nodata, log_level, fail):
_setupLogging(log_level)
reader, writer = setupIO(nodata)
@@ -117,25 +111,28 @@ def main(config, data, flagger, outfile, nodata, log_level, fail):
saqc = SaQC(
data=data,
nodata=nodata,
+ translator=SCHEMES[scheme or "float"](),
error_policy="raise" if fail else "warn",
)
- data_result, flags_result = saqc.readConfig(config).getResult(raw=True)
+ data_result, flags_result = saqc.readConfig(config).getResult()
if outfile:
- data_frame = data_result.to_df()
- flags_frame = flags_result.toFrame()
- unflagged = (flags_frame == UNFLAGGED) | flags_frame.isna()
- flags_frame[unflagged] = GOOD
- fields = {"data": data_frame, "flags": flags_frame}
+ data_result.columns = pd.MultiIndex.from_product(
+ [data_result.columns.tolist(), ["data"]]
+ )
+
+ if not isinstance(flags_result.columns, pd.MultiIndex):
+ flags_result.columns = pd.MultiIndex.from_product(
+ [flags_result.columns.tolist(), ["flags"]]
+ )
out = (
- pd.concat(fields.values(), axis=1, keys=fields.keys())
- .reorder_levels(order=[1, 0], axis=1)
+ pd.concat([data_result, flags_result], axis=1)
.sort_index(axis=1, level=0, sort_remaining=False)
)
- out.columns = out.columns.rename(["", ""])
+
writeData(writer, out, outfile)
diff --git a/saqc/core/translator/dmptranslator.py b/saqc/core/translator/dmptranslator.py
index 1f173b691..5f2f7d1cd 100644
--- a/saqc/core/translator/dmptranslator.py
+++ b/saqc/core/translator/dmptranslator.py
@@ -2,6 +2,7 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
+from dios.dios.dios import DictOfSeries
import json
from typing import List, Tuple
@@ -137,7 +138,6 @@ class DmpTranslator(Translator):
tflags = super().backward(flags, call_graph)
out = {}
- # import pdb; pdb.set_trace()
for field in tflags.columns:
flag_call_history = self._getFieldFunctions(field, call_graph)
flag_pos = flags.history[field].idxmax()
@@ -158,8 +158,8 @@ class DmpTranslator(Translator):
var_flags = {
"quality_flag": tflags[field],
- "quality_comment": comments,
- "quality_cause": causes,
+ "quality_comment": pd.Series(comments, index=flags[field].index),
+ "quality_cause": pd.Series(causes, index=flags[field].index),
}
out[field] = pd.DataFrame(var_flags)
return pd.concat(out, axis="columns")
diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py
index 019ab4f37..c4e0bb046 100644
--- a/tests/integration/test_integration.py
+++ b/tests/integration/test_integration.py
@@ -8,17 +8,18 @@ def test__main__py():
# if not run from project root
projpath = os.path.dirname(saqc.__file__) + "/../"
-
+ args = [
+ "--config",
+ projpath + "ressources/data/config_ci.csv",
+ "--data",
+ projpath + "ressources/data/data.csv",
+ "--outfile",
+ "/tmp/test.csv", # the filesystem temp dir
+ ]
runner = CliRunner()
- result = runner.invoke(
- saqc.__main__.main,
- [
- "--config",
- projpath + "ressources/data/config_ci.csv",
- "--data",
- projpath + "ressources/data/data.csv",
- "--outfile",
- "/tmp/test.csv", # the filesystem temp dir
- ],
- )
+ result = runner.invoke(saqc.__main__.main, args)
assert result.exit_code == 0, result.output
+
+ for scheme in ["float", "dmp", "positional"]:
+ result = runner.invoke(saqc.__main__.main, args + ["--scheme", scheme])
+ assert result.exit_code == 0, result.output
--
GitLab
From 8edc193c993abfca2d9761d7f29ebe0223e03cdb Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Tue, 4 May 2021 14:51:16 +0200
Subject: [PATCH 141/180] cleanups
---
saqc/__init__.py | 10 +++++++++-
saqc/__main__.py | 5 ++---
saqc/core/core.py | 3 ++-
saqc/core/flags.py | 8 ++++----
saqc/core/translator/basetranslator.py | 21 ++++++++------------
saqc/core/translator/dmptranslator.py | 4 +++-
saqc/core/translator/positionaltranslator.py | 8 ++++----
tests/core/test_translator.py | 4 +---
8 files changed, 33 insertions(+), 30 deletions(-)
diff --git a/saqc/__init__.py b/saqc/__init__.py
index 7fa49e8dd..295297f36 100644
--- a/saqc/__init__.py
+++ b/saqc/__init__.py
@@ -5,4 +5,12 @@ __version__ = "1.4"
# import order: from small to big
from saqc.constants import *
-from saqc.core import register, initFlagsLike, Flags, SaQC, FloatTranslator, DmpTranslator, PositionalTranslator
+from saqc.core import (
+ register,
+ initFlagsLike,
+ Flags,
+ SaQC,
+ FloatTranslator,
+ DmpTranslator,
+ PositionalTranslator,
+)
diff --git a/saqc/__main__.py b/saqc/__main__.py
index 50dab15cc..d26f9d3c3 100644
--- a/saqc/__main__.py
+++ b/saqc/__main__.py
@@ -128,9 +128,8 @@ def main(config, data, scheme, outfile, nodata, log_level, fail):
[flags_result.columns.tolist(), ["flags"]]
)
- out = (
- pd.concat([data_result, flags_result], axis=1)
- .sort_index(axis=1, level=0, sort_remaining=False)
+ out = pd.concat([data_result, flags_result], axis=1).sort_index(
+ axis=1, level=0, sort_remaining=False
)
writeData(writer, out, outfile)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index bc31aa81f..64d9b1faf 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -134,13 +134,14 @@ class SaQC(FuncModules):
self._flags = self._initFlags(data, flags)
self._error_policy = error_policy
self._translator = translator or FloatTranslator()
+
# NOTE:
# We need two lists to represent the future and the past computations
# on a `SaQC`-Object. Due to the dynamic nature of field expansion
# with regular expressions, we can't just reuse the original execution
# plan to infer all translation related information.
self._planned: CallGraph = [] # will be filled by calls to `_wrap`
- self._computed: MaterializedGraph = self._translator._buildCallGraph(
+ self._computed: MaterializedGraph = self._translator.buildGraph(
self._flags
) # will be filled in `evaluate`
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index 15e5f6f93..b49647d1d 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -186,10 +186,10 @@ class Flags:
# There are not to may write access possibilities here so we don't
# have to much trouble.
# NOTE:
- # `time pytest tests/core tests/funcs tests/integration` yields
- # identical runtimes without or without the cache, the cache however
- # adds code complexity, through the additional dictionary and stuff
- # like the `_HistAccess`. If tests on real-world datasets give
+ # `time pytest tests/core tests/funcs tests/integration tests/fuzzy`
+ # yields # identical runtimes without or without the cache, the cache
+ # however adds code complexity, through the additional dictionary and
+ # stuff like the `_HistAccess`. If tests on real-world datasets give
# similar results, we should get rid of it.
self._cache = {}
diff --git a/saqc/core/translator/basetranslator.py b/saqc/core/translator/basetranslator.py
index 32b8d55cf..d70c88bc2 100644
--- a/saqc/core/translator/basetranslator.py
+++ b/saqc/core/translator/basetranslator.py
@@ -1,10 +1,6 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-# TODO:
-# - remove `MaterializedGraph` from `Translator.forward`
-# - fix exe
-
from __future__ import annotations
from saqc.core.lib import SaQCFunction, ColumnSelector
@@ -20,7 +16,7 @@ from saqc.core.flags import (
UNFLAGGED,
BAD,
)
-from saqc.lib.types import ExternalFlag, MaterializedGraph
+from saqc.lib.types import ExternalFlag, MaterializedGraph, DiosLikeT
ForwardMap = Dict[ExternalFlag, float]
@@ -153,19 +149,19 @@ class Translator:
return mapFlags
@staticmethod
- def _buildCallGraph(flags: Flags) -> MaterializedGraph:
+ def buildGraph(flags: Flags) -> MaterializedGraph:
"""
- build a call graph from the `Flags` and their `History`
+ build a call graph from the external flags
+
+ Build an `MaterializedGraph`, that can be used
+ in replays of the original `SaQC` run yielding the
+ same result for the same input data set.
As we usually don't have enough information (i.e. SaQC
function name and all used parameters) we generate dummy
functions here. These dummy functions unconditionally set
the `field` to the provided flags.
- The idea is, to spit out an `MaterializedGraph`, that can
- be used in replays of the original `SaQC` run in gives the
- same result for the same input data set.
-
Parameters
----------
flags : flags to generate a call graph for
@@ -201,8 +197,7 @@ class Translator:
Flags object
"""
tflags = Flags(self._translate(flags, self._forward))
- graph = self._buildCallGraph(tflags)
- return tflags, graph
+ return tflags, self.buildGraph(tflags)
def backward(self, flags: Flags, call_graph: MaterializedGraph) -> pd.DataFrame:
"""
diff --git a/saqc/core/translator/dmptranslator.py b/saqc/core/translator/dmptranslator.py
index 5f2f7d1cd..f5397ab51 100644
--- a/saqc/core/translator/dmptranslator.py
+++ b/saqc/core/translator/dmptranslator.py
@@ -105,7 +105,9 @@ class DmpTranslator(Translator):
# we can't infer information about the ordering of function calls,
# so we order the history by appearance
# for _, group in data.fillna("").groupby(["test", "comment", "causes"]):
- for _, group in data.loc[data["test"].replace("", np.nan).notna()].groupby(["test", "comment", "causes"]):
+ for _, group in data.loc[data["test"].replace("", np.nan).notna()].groupby(
+ ["test", "comment", "causes"]
+ ):
fname, comment, cause = group.iloc[0]
func = SaQCFunction(
name=fname,
diff --git a/saqc/core/translator/positionaltranslator.py b/saqc/core/translator/positionaltranslator.py
index dae62477b..5db84c516 100644
--- a/saqc/core/translator/positionaltranslator.py
+++ b/saqc/core/translator/positionaltranslator.py
@@ -16,7 +16,8 @@ from saqc.core.flags import (
BAD,
)
from saqc.lib.types import MaterializedGraph
-from saqc.core.translator.basetranslator import Translator, ForwardMap
+from saqc.core.translator.basetranslator import Translator, ForwardMap, BackwardMap
+
class PositionalTranslator(Translator):
@@ -24,7 +25,7 @@ class PositionalTranslator(Translator):
Implements the translation from and to the flagging scheme implemented by CHS
"""
- _FORWARD: ForwardMap = {0: UNFLAGGED, 1: DOUBTFUL, 2: BAD}
+ _FORWARD: ForwardMap = {0: UNFLAGGED, 1: DOUBTFUL, 2: BAD}
_BACKWARD: BackwardMap = {
UNTOUCHED: 0,
UNFLAGGED: 0,
@@ -65,8 +66,7 @@ class PositionalTranslator(Translator):
data[field] = field_history
tflags = Flags(data)
- graph = Translator._buildCallGraph(tflags)
-
+ graph = self.buildGraph(tflags)
return tflags, graph
def backward(self, flags: Flags, call_stack: MaterializedGraph) -> pd.DataFrame:
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index 67ba5134f..4cca96a31 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -158,8 +158,6 @@ def test_positionalTranslatorIntegration():
for field in flags.columns:
assert flags[field].astype(str).str.match("^9[012]*$").all()
- assert (flags[col].astype(str).str.len() == 3).all()
-
round_trip = translator.backward(*translator.forward(flags))
assert (flags.values == round_trip.values).all()
@@ -252,7 +250,7 @@ def test_callHistoryYieldsSameResults():
# generate a dummy call history from flags
translator = FloatTranslator()
- graph = translator._buildCallGraph(flags1)
+ graph = translator.buildGraph(flags1)
saqc2 = SaQC(data=data)
# convert the call history into an excution plan and inject into a blank SaQC object
--
GitLab
From 84e859b4d76fa652aba04b4b84daef1b42dd11fc Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 6 May 2021 12:43:52 +0200
Subject: [PATCH 142/180] [FIX] don't overwrite columns of input history
---
saqc/core/history.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/saqc/core/history.py b/saqc/core/history.py
index fa3491a1c..8e92243fc 100644
--- a/saqc/core/history.py
+++ b/saqc/core/history.py
@@ -253,8 +253,9 @@ class History:
raise ValueError("Index does not match")
n = len(self.columns)
- value_hist = value.hist
- value_mask = value.mask
+ # don't overwrite the `.columns` of the input down the line
+ value_hist = value.hist.copy(deep=False)
+ value_mask = value.mask.copy(deep=False)
if not force:
value_hist = value_hist.iloc[:, n:]
--
GitLab
From 92c1a054d33f2967ae4d657c93242bd230869648 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 6 May 2021 12:44:39 +0200
Subject: [PATCH 143/180] [FIX] history merges in mapToOriginal may lead to an
inconsistent history mask
---
saqc/funcs/resampling.py | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 97e8cba0b..be2c38167 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -740,5 +740,17 @@ def reindexFlags(
history = applyFunctionOnHistory(
flags.history[source], func, func_kws, func, mask_kws, last_column=dummy
)
+ # NOTE:
+ # We need to mask the last column in the original History, because the
+ # following workflow produces an incompletely masked History otherwise:
+ #
+ # saqc
+ # .flagFoo(var) -> var-history gets a first unmasked column
+ # .shift(var) -> we add a copy of var + its unmasked history
+ # .flagBar(var) -> the shifted var-history gets a second column
+ # .mapToOriginal(var) -> we combine both var-Histories (var_original and
+ # var) with an unmasked column each with an
+ # resulting flag from flagFoo
+ flags.history[field].mask.iloc[:, -1] = False
flags.history[field] = flags.history[field].append(history, force=False)
return data, flags
--
GitLab
From 663155bd5b2ac1b061cdc01c5b5d820ab3ea66b5 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 6 May 2021 17:38:21 +0200
Subject: [PATCH 144/180] undo broken [FIX]
---
saqc/funcs/resampling.py | 12 ------------
1 file changed, 12 deletions(-)
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index be2c38167..97e8cba0b 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -740,17 +740,5 @@ def reindexFlags(
history = applyFunctionOnHistory(
flags.history[source], func, func_kws, func, mask_kws, last_column=dummy
)
- # NOTE:
- # We need to mask the last column in the original History, because the
- # following workflow produces an incompletely masked History otherwise:
- #
- # saqc
- # .flagFoo(var) -> var-history gets a first unmasked column
- # .shift(var) -> we add a copy of var + its unmasked history
- # .flagBar(var) -> the shifted var-history gets a second column
- # .mapToOriginal(var) -> we combine both var-Histories (var_original and
- # var) with an unmasked column each with an
- # resulting flag from flagFoo
- flags.history[field].mask.iloc[:, -1] = False
flags.history[field] = flags.history[field].append(history, force=False)
return data, flags
--
GitLab
From 01a73f0c92aac3f8e2aeb02712d664a427d1d08d Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 6 May 2021 17:39:20 +0200
Subject: [PATCH 145/180] [FIX] history merges may lead to an inconsistent mask
---
saqc/core/history.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/saqc/core/history.py b/saqc/core/history.py
index 8e92243fc..23f660e34 100644
--- a/saqc/core/history.py
+++ b/saqc/core/history.py
@@ -266,6 +266,9 @@ class History:
value_hist.columns = columns
value_mask.columns = columns
+ # clear the current mask
+ self.mask.loc[(~value_mask & value_hist.notna()).any(axis="columns")] = False
+
self.hist.loc[:, columns] = value_hist.copy()
self.mask.loc[:, columns] = value_mask.copy()
return self
--
GitLab
From 0fc94e150850df6d8c6b650af7df573b829864cc Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 6 May 2021 21:24:41 +0200
Subject: [PATCH 146/180] [FIX] pass actual flags into generic
---
saqc/core/register.py | 7 -------
saqc/funcs/generic.py | 33 ++++++++++++++++++---------------
2 files changed, 18 insertions(+), 22 deletions(-)
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 624e033c9..80a06a0db 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -210,13 +210,6 @@ def _getMaskingThresh(masking, kwargs, fname):
if not isinstance(thresh, (bool, float, int)):
raise TypeError(f"'to_mask' must be of type bool or float")
- if masking == "none" and thresh not in (False, np.inf):
- # TODO: fix warning reference to docu
- warnings.warn(
- f"the saqc-function {fname!r} ignores masking and therefore does not evaluate the passed "
- f"'to_mask'-keyword. Please refer to the documentation: TODO"
- )
-
if thresh is True: # masking ON
thresh = UNFLAGGED
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index 86cb2b572..2088196e7 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -10,8 +10,9 @@ import pandas as pd
from dios import DictOfSeries
-from saqc.constants import *
-from saqc.core import register, initFlagsLike, Flags
+from saqc.constants import GOOD, BAD, UNFLAGGED
+from saqc.core.flags import initFlagsLike, Flags
+from saqc.core.register import register, _maskData
from saqc.core.visitor import ENVIRONMENT
import operator as op
@@ -84,13 +85,14 @@ def _execGeneric(
return func(*args)
-@register(masking="all", module="generic")
+@register(masking="none", module="generic")
def process(
data: DictOfSeries,
field: str,
flags: Flags,
func: Callable[[pd.Series], pd.Series],
nodata: float = np.nan,
+ to_mask: float = UNFLAGGED,
**kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
@@ -142,18 +144,20 @@ def process(
>>> lambda temperature, uncertainty: np.round(temperature) * np.sqrt(uncertainty)
"""
- data[field] = _execGeneric(flags, data, func, field, nodata).squeeze()
+ # we get the data unmaskes in order to also receive flags,
+ # so let's ge to the masking manually
+ data_masked, _ = _maskData(data, flags, data.columns, to_mask)
+ data[field] = _execGeneric(flags, data_masked, func, field, nodata).squeeze()
- # TODO: the former comment wished to overwrite the column, but i'm not sure -- palmb
- # see #GL177
if field in flags:
flags.drop(field)
flags[field] = initFlagsLike(data[field])[field]
+
return data, flags
-@register(masking="all", module="generic")
+@register(masking="none", module="generic")
def flag(
data: DictOfSeries,
field: str,
@@ -161,6 +165,7 @@ def flag(
func: Callable[[pd.Series], pd.Series],
nodata: float = np.nan,
flag: float = BAD,
+ to_mask: float = UNFLAGGED,
**kwargs,
) -> Tuple[DictOfSeries, Flags]:
# TODO : fix docstring, check if all still works
@@ -240,10 +245,11 @@ def flag(
>>> lambda level: np.sqrt(level) > 7
"""
- # NOTE:
- # The naming of the func parameter is pretty confusing
- # as it actually holds the result of a generic expression
- mask = _execGeneric(flags, data, func, field, nodata).squeeze()
+ # we get the data unmaskes in order to also receive flags,
+ # so let's ge to the masking manually
+ data_masked, _ = _maskData(data, flags, data.columns, to_mask)
+
+ mask = _execGeneric(flags, data_masked, func, field, nodata).squeeze()
if np.isscalar(mask):
raise TypeError(f"generic expression does not return an array")
if not np.issubdtype(mask.dtype, np.bool_):
@@ -252,9 +258,6 @@ def flag(
if field not in flags.columns:
flags[field] = pd.Series(UNFLAGGED, index=mask.index, name=field)
- # if flags.getFlags(field).empty:
- # flags = flags.merge(
- # flags.initFlags(
- # data=pd.Series(name=field, index=mask.index, dtype=np.float64)))
flags[mask, field] = flag
+
return data, flags
--
GitLab
From 9d51a0672fc0919aae335aba9570b26eabcc8e3f Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 6 May 2021 21:25:43 +0200
Subject: [PATCH 147/180] [FIX] flagtools.flagUnflagged did not see the actual
flags
---
saqc/funcs/flagtools.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index 29ccf84a0..c675d1830 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -82,7 +82,7 @@ def clearFlags(
return forceFlags(data, field, flags, flag=UNFLAGGED, **kwargs)
-@register(masking="field", module="flagtools")
+@register(masking="none", module="flagtools")
def flagUnflagged(
data: DictOfSeries, field: ColumnName, flags: Flags, flag: float = BAD, **kwargs
) -> Tuple[DictOfSeries, Flags]:
--
GitLab
From 5c97a4dec56daf611013ae4f61b83d013aa09d3c Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 6 May 2021 21:32:42 +0200
Subject: [PATCH 148/180] [FIX] breaks.flagIsolated was only flaggin nan values
---
saqc/funcs/breaks.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/saqc/funcs/breaks.py b/saqc/funcs/breaks.py
index ff6af9e8b..e14d6826f 100644
--- a/saqc/funcs/breaks.py
+++ b/saqc/funcs/breaks.py
@@ -126,16 +126,21 @@ def flagIsolated(
bools = pd.Series(data=0, index=mask.index, dtype=bool)
for srs in groupConsecutives(mask):
if np.all(~srs):
+ # we found a chunk of non-nan values
start = srs.index[0]
stop = srs.index[-1]
if stop - start <= group_window:
+ # the chunk is large enough
left = mask[start - gap_window : start].iloc[:-1]
if left.all():
+ # the section before our chunk is nan-only
right = mask[stop : stop + gap_window].iloc[1:]
if right.all():
+ # the section after our chunk is nan-only
+ # -> we found a chunk of isolated non-values
bools[start:stop] = True
- flags[mask, field] = flag
+ flags[bools, field] = flag
return data, flags
--
GitLab
From ea06005f013b77b676ddd61dcde8dcb996fe7745 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 6 May 2021 12:43:52 +0200
Subject: [PATCH 149/180] [FIX] don't overwrite columns of input history
---
saqc/core/history.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/saqc/core/history.py b/saqc/core/history.py
index c262d506f..0668d68d1 100644
--- a/saqc/core/history.py
+++ b/saqc/core/history.py
@@ -247,8 +247,9 @@ class History:
raise ValueError("Index does not match")
n = len(self.columns)
- value_hist = value.hist
- value_mask = value.mask
+ # don't overwrite the `.columns` of the input down the line
+ value_hist = value.hist.copy(deep=False)
+ value_mask = value.mask.copy(deep=False)
if not force:
value_hist = value_hist.iloc[:, n:]
--
GitLab
From b3a98afe3c2e088d7f48a7e8faaa7476107bee56 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 6 May 2021 17:39:20 +0200
Subject: [PATCH 150/180] [FIX] history merges may lead to an inconsistent mask
---
saqc/core/history.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/saqc/core/history.py b/saqc/core/history.py
index 0668d68d1..79b78b579 100644
--- a/saqc/core/history.py
+++ b/saqc/core/history.py
@@ -260,6 +260,9 @@ class History:
value_hist.columns = columns
value_mask.columns = columns
+ # clear the current mask
+ self.mask.loc[(~value_mask & value_hist.notna()).any(axis="columns")] = False
+
self.hist.loc[:, columns] = value_hist.copy()
self.mask.loc[:, columns] = value_mask.copy()
return self
--
GitLab
From 65cd7e3003aa8a6acb02e5435b01b278dbd9f5cb Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 6 May 2021 21:24:41 +0200
Subject: [PATCH 151/180] [FIX] pass actual flags into generic
---
saqc/core/register.py | 7 -------
saqc/funcs/generic.py | 33 ++++++++++++++++++---------------
2 files changed, 18 insertions(+), 22 deletions(-)
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 624e033c9..80a06a0db 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -210,13 +210,6 @@ def _getMaskingThresh(masking, kwargs, fname):
if not isinstance(thresh, (bool, float, int)):
raise TypeError(f"'to_mask' must be of type bool or float")
- if masking == "none" and thresh not in (False, np.inf):
- # TODO: fix warning reference to docu
- warnings.warn(
- f"the saqc-function {fname!r} ignores masking and therefore does not evaluate the passed "
- f"'to_mask'-keyword. Please refer to the documentation: TODO"
- )
-
if thresh is True: # masking ON
thresh = UNFLAGGED
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index 86cb2b572..2088196e7 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -10,8 +10,9 @@ import pandas as pd
from dios import DictOfSeries
-from saqc.constants import *
-from saqc.core import register, initFlagsLike, Flags
+from saqc.constants import GOOD, BAD, UNFLAGGED
+from saqc.core.flags import initFlagsLike, Flags
+from saqc.core.register import register, _maskData
from saqc.core.visitor import ENVIRONMENT
import operator as op
@@ -84,13 +85,14 @@ def _execGeneric(
return func(*args)
-@register(masking="all", module="generic")
+@register(masking="none", module="generic")
def process(
data: DictOfSeries,
field: str,
flags: Flags,
func: Callable[[pd.Series], pd.Series],
nodata: float = np.nan,
+ to_mask: float = UNFLAGGED,
**kwargs,
) -> Tuple[DictOfSeries, Flags]:
"""
@@ -142,18 +144,20 @@ def process(
>>> lambda temperature, uncertainty: np.round(temperature) * np.sqrt(uncertainty)
"""
- data[field] = _execGeneric(flags, data, func, field, nodata).squeeze()
+ # we get the data unmaskes in order to also receive flags,
+ # so let's ge to the masking manually
+ data_masked, _ = _maskData(data, flags, data.columns, to_mask)
+ data[field] = _execGeneric(flags, data_masked, func, field, nodata).squeeze()
- # TODO: the former comment wished to overwrite the column, but i'm not sure -- palmb
- # see #GL177
if field in flags:
flags.drop(field)
flags[field] = initFlagsLike(data[field])[field]
+
return data, flags
-@register(masking="all", module="generic")
+@register(masking="none", module="generic")
def flag(
data: DictOfSeries,
field: str,
@@ -161,6 +165,7 @@ def flag(
func: Callable[[pd.Series], pd.Series],
nodata: float = np.nan,
flag: float = BAD,
+ to_mask: float = UNFLAGGED,
**kwargs,
) -> Tuple[DictOfSeries, Flags]:
# TODO : fix docstring, check if all still works
@@ -240,10 +245,11 @@ def flag(
>>> lambda level: np.sqrt(level) > 7
"""
- # NOTE:
- # The naming of the func parameter is pretty confusing
- # as it actually holds the result of a generic expression
- mask = _execGeneric(flags, data, func, field, nodata).squeeze()
+ # we get the data unmaskes in order to also receive flags,
+ # so let's ge to the masking manually
+ data_masked, _ = _maskData(data, flags, data.columns, to_mask)
+
+ mask = _execGeneric(flags, data_masked, func, field, nodata).squeeze()
if np.isscalar(mask):
raise TypeError(f"generic expression does not return an array")
if not np.issubdtype(mask.dtype, np.bool_):
@@ -252,9 +258,6 @@ def flag(
if field not in flags.columns:
flags[field] = pd.Series(UNFLAGGED, index=mask.index, name=field)
- # if flags.getFlags(field).empty:
- # flags = flags.merge(
- # flags.initFlags(
- # data=pd.Series(name=field, index=mask.index, dtype=np.float64)))
flags[mask, field] = flag
+
return data, flags
--
GitLab
From 1da172e80599b4ac52573239f2017f28cab549b7 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 6 May 2021 21:25:43 +0200
Subject: [PATCH 152/180] [FIX] flagtools.flagUnflagged did not see the actual
flags
---
saqc/funcs/flagtools.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index 29ccf84a0..c675d1830 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -82,7 +82,7 @@ def clearFlags(
return forceFlags(data, field, flags, flag=UNFLAGGED, **kwargs)
-@register(masking="field", module="flagtools")
+@register(masking="none", module="flagtools")
def flagUnflagged(
data: DictOfSeries, field: ColumnName, flags: Flags, flag: float = BAD, **kwargs
) -> Tuple[DictOfSeries, Flags]:
--
GitLab
From 0119d7e10c8b305d79886cb3059bb56ac3177035 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 6 May 2021 21:32:42 +0200
Subject: [PATCH 153/180] [FIX] breaks.flagIsolated was only flaggin nan values
---
saqc/funcs/breaks.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/saqc/funcs/breaks.py b/saqc/funcs/breaks.py
index ff6af9e8b..e14d6826f 100644
--- a/saqc/funcs/breaks.py
+++ b/saqc/funcs/breaks.py
@@ -126,16 +126,21 @@ def flagIsolated(
bools = pd.Series(data=0, index=mask.index, dtype=bool)
for srs in groupConsecutives(mask):
if np.all(~srs):
+ # we found a chunk of non-nan values
start = srs.index[0]
stop = srs.index[-1]
if stop - start <= group_window:
+ # the chunk is large enough
left = mask[start - gap_window : start].iloc[:-1]
if left.all():
+ # the section before our chunk is nan-only
right = mask[stop : stop + gap_window].iloc[1:]
if right.all():
+ # the section after our chunk is nan-only
+ # -> we found a chunk of isolated non-values
bools[start:stop] = True
- flags[mask, field] = flag
+ flags[bools, field] = flag
return data, flags
--
GitLab
From 6143c92eda485d97e7154ba1618a0b85ef99a9a3 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Fri, 7 May 2021 09:10:03 +0200
Subject: [PATCH 154/180] [FIX] excess history column for new variables through
generic.flag
---
saqc/funcs/generic.py | 15 ++++++---------
1 file changed, 6 insertions(+), 9 deletions(-)
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index 2088196e7..06cc8dfe9 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -3,7 +3,7 @@
from functools import partial
from inspect import signature
-from typing import Tuple, Any, Union, Callable
+from typing import Tuple, Union, Callable
import numpy as np
import pandas as pd
@@ -144,8 +144,8 @@ def process(
>>> lambda temperature, uncertainty: np.round(temperature) * np.sqrt(uncertainty)
"""
- # we get the data unmaskes in order to also receive flags,
- # so let's ge to the masking manually
+ # we get the data unmasked in order to also receive flags,
+ # so let's do to the masking manually
data_masked, _ = _maskData(data, flags, data.columns, to_mask)
data[field] = _execGeneric(flags, data_masked, func, field, nodata).squeeze()
@@ -245,8 +245,8 @@ def flag(
>>> lambda level: np.sqrt(level) > 7
"""
- # we get the data unmaskes in order to also receive flags,
- # so let's ge to the masking manually
+ # we get the data unmasked, in order to also receive flags,
+ # so let's do to the masking manually
data_masked, _ = _maskData(data, flags, data.columns, to_mask)
mask = _execGeneric(flags, data_masked, func, field, nodata).squeeze()
@@ -255,9 +255,6 @@ def flag(
if not np.issubdtype(mask.dtype, np.bool_):
raise TypeError(f"generic expression does not return a boolean array")
- if field not in flags.columns:
- flags[field] = pd.Series(UNFLAGGED, index=mask.index, name=field)
-
- flags[mask, field] = flag
+ flags[field] = mask.replace({False: UNFLAGGED, True: BAD})
return data, flags
--
GitLab
From 3aa1804d49f4a2fdc4ea5132238c8625aae4c65b Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Fri, 7 May 2021 09:23:20 +0200
Subject: [PATCH 155/180] [FIX] outliers.flagOffset raises IndexError for all
nan-data
---
saqc/funcs/outliers.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index e3d473ef7..14a4bb755 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -875,6 +875,8 @@ def flagOffset(
"""
dataseries = data[field].dropna()
+ if dataseries.empty:
+ return data, flags
# using reverted series - because ... long story.
ind = dataseries.index
--
GitLab
From f454ae07ce91e3d3f761ce3a45e3ed608ec437d7 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Fri, 7 May 2021 09:55:54 +0200
Subject: [PATCH 156/180] [FIX] remove real-world memory usage by ~20% and
runtime by ~3% through removal of Flags._cache
---
saqc/core/flags.py | 32 ++++----------------------------
tests/core/test_flags.py | 33 ---------------------------------
2 files changed, 4 insertions(+), 61 deletions(-)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index b49647d1d..63c48508b 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -48,7 +48,6 @@ class _HistAccess:
self.obj._validateHistForFlags(value)
self.obj._data[key] = value
- self.obj._cache.pop(key, None)
class Flags:
@@ -180,19 +179,6 @@ class Flags:
else:
self._data = self._initFromRaw(raw_data, copy)
- # this is a simple cache that reduce the calculation of the flags
- # from the entire history of a flag column. The _cache is filled
- # with __getitem__ and cleared on any write access to self_data.
- # There are not to may write access possibilities here so we don't
- # have to much trouble.
- # NOTE:
- # `time pytest tests/core tests/funcs tests/integration tests/fuzzy`
- # yields # identical runtimes without or without the cache, the cache
- # however adds code complexity, through the additional dictionary and
- # stuff like the `_HistAccess`. If tests on real-world datasets give
- # similar results, we should get rid of it.
- self._cache = {}
-
@staticmethod
def _initFromRaw(data: Mapping, copy: bool) -> Dict[str, History]:
"""
@@ -282,16 +268,12 @@ class Flags:
if not len(value) == len(self):
raise ValueError("index must match current index in length")
- _data, _cache = {}, {}
+ _data = {}
for old, new in zip(self.columns, value):
_data[new] = self._data[old]
- if old in self._cache:
- _cache[new] = self._cache[old]
-
self._data = _data
- self._cache = _cache
@property
def empty(self) -> bool:
@@ -315,11 +297,7 @@ class Flags:
# item access
def __getitem__(self, key: str) -> pd.Series:
-
- if key not in self._cache:
- self._cache[key] = self._data[key].max()
-
- return self._cache[key].copy()
+ return self._data[key].max()
def __setitem__(self, key: SelectT, value: ValueT):
# force-KW is only internally available
@@ -365,11 +343,9 @@ class Flags:
self._data[key] = _simpleHist(value.index)
self._data[key].append(value, force=True)
- self._cache.pop(key, None)
def __delitem__(self, key):
self._data.pop(key)
- self._cache.pop(key, None)
def drop(self, key: str):
"""
@@ -455,8 +431,8 @@ class Flags:
"""
di = dios.DictOfSeries(columns=self.columns)
- for k, v in self._data.items():
- di[k] = self[k] # use cache
+ for k in self._data.keys():
+ di[k] = self[k]
return di.copy()
diff --git a/tests/core/test_flags.py b/tests/core/test_flags.py
index 584999bc9..7bc08c007 100644
--- a/tests/core/test_flags.py
+++ b/tests/core/test_flags.py
@@ -237,39 +237,6 @@ def test_set_flags_with_index(
flags[index, c] = wrong_len
-def test_cache():
- arr = np.array(
- [
- [0, 0, 0, 0],
- [0, 1, 2, 3],
- [0, 1, 2, 3],
- ]
- )
- data = pd.DataFrame(arr, dtype=float, columns=list("abcd"))
- flags = Flags(data)
-
- # cache empty
- assert flags._cache == {}
-
- # invoke caching
- flags["a"]
- assert "a" in flags._cache
-
- # clears cache
- flags["a"] = pd.Series([0, 0, 0], dtype=float)
- assert "a" not in flags._cache
-
- # cache all
- flags.toDios()
- for c in flags.columns:
- assert c in flags._cache
-
- # cache survive renaming
- flags.columns = list("xyzq")
- for c in flags.columns:
- assert c in flags._cache
-
-
def _validate_flags_equals_frame(flags, df):
assert df.columns.equals(flags.columns)
--
GitLab
From b2137c2ac63f37bdea76a345d7721d70f04a03df Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Fri, 7 May 2021 09:10:03 +0200
Subject: [PATCH 157/180] [FIX] excess history column for new variables through
generic.flag
---
saqc/funcs/generic.py | 15 ++++++---------
1 file changed, 6 insertions(+), 9 deletions(-)
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index 2088196e7..06cc8dfe9 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -3,7 +3,7 @@
from functools import partial
from inspect import signature
-from typing import Tuple, Any, Union, Callable
+from typing import Tuple, Union, Callable
import numpy as np
import pandas as pd
@@ -144,8 +144,8 @@ def process(
>>> lambda temperature, uncertainty: np.round(temperature) * np.sqrt(uncertainty)
"""
- # we get the data unmaskes in order to also receive flags,
- # so let's ge to the masking manually
+ # we get the data unmasked in order to also receive flags,
+ # so let's do to the masking manually
data_masked, _ = _maskData(data, flags, data.columns, to_mask)
data[field] = _execGeneric(flags, data_masked, func, field, nodata).squeeze()
@@ -245,8 +245,8 @@ def flag(
>>> lambda level: np.sqrt(level) > 7
"""
- # we get the data unmaskes in order to also receive flags,
- # so let's ge to the masking manually
+ # we get the data unmasked, in order to also receive flags,
+ # so let's do to the masking manually
data_masked, _ = _maskData(data, flags, data.columns, to_mask)
mask = _execGeneric(flags, data_masked, func, field, nodata).squeeze()
@@ -255,9 +255,6 @@ def flag(
if not np.issubdtype(mask.dtype, np.bool_):
raise TypeError(f"generic expression does not return a boolean array")
- if field not in flags.columns:
- flags[field] = pd.Series(UNFLAGGED, index=mask.index, name=field)
-
- flags[mask, field] = flag
+ flags[field] = mask.replace({False: UNFLAGGED, True: BAD})
return data, flags
--
GitLab
From 734621f724a8f0bfcf8bf91ce1a6f8d952ce7cd5 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Fri, 7 May 2021 09:23:20 +0200
Subject: [PATCH 158/180] [FIX] outliers.flagOffset raises IndexError for all
nan-data
---
saqc/funcs/outliers.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index e3d473ef7..14a4bb755 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -875,6 +875,8 @@ def flagOffset(
"""
dataseries = data[field].dropna()
+ if dataseries.empty:
+ return data, flags
# using reverted series - because ... long story.
ind = dataseries.index
--
GitLab
From e880fa7daaf8be07d2099cd575ca14f982782f99 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Fri, 7 May 2021 09:55:54 +0200
Subject: [PATCH 159/180] [FIX] remove real-world memory usage by ~20% and
runtime by ~3% through removal of Flags._cache
---
saqc/core/flags.py | 29 ++++++-----------------------
tests/core/test_flags.py | 33 ---------------------------------
2 files changed, 6 insertions(+), 56 deletions(-)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index 34b2fed23..af226a768 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -45,7 +45,6 @@ class _HistAccess:
History._validateHistWithMask(value.hist, value.mask)
self.obj._data[key] = value
- self.obj._cache.pop(key, None)
class Flags:
@@ -175,14 +174,8 @@ class Flags:
# with python 3.7 dicts are insertion-ordered by default
self._data = self._initFromRaw(raw_data, copy)
- # this is a simple cache that reduce the calculation of the flags
- # from the entire history of a flag column. The _cache is filled
- # with __getitem__ and cleared on any write access to self_data.
- # There are not to may write access possibilities here so we don't
- # have to much trouble.
- self._cache = {}
-
- def _initFromRaw(self, data, copy) -> Dict[str, History]:
+ @staticmethod
+ def _initFromRaw(data: Mapping, copy: bool) -> Dict[str, History]:
"""
init from dict-like: keys are flag column, values become
initial columns of history(s).
@@ -250,16 +243,12 @@ class Flags:
if not len(value) == len(self):
raise ValueError("index must match current index in length")
- _data, _cache = {}, {}
+ _data = {}
for old, new in zip(self.columns, value):
_data[new] = self._data[old]
- if old in self._cache:
- _cache[new] = self._cache[old]
-
self._data = _data
- self._cache = _cache
@property
def empty(self) -> bool:
@@ -283,11 +272,7 @@ class Flags:
# item access
def __getitem__(self, key: str) -> pd.Series:
-
- if key not in self._cache:
- self._cache[key] = self._data[key].max()
-
- return self._cache[key].copy()
+ return self._data[key].max()
def __setitem__(self, key: SelectT, value: ValueT):
# force-KW is internal available only
@@ -330,11 +315,9 @@ class Flags:
self._data[key] = History()
self._data[key].append(value, force=True)
- self._cache.pop(key, None)
def __delitem__(self, key):
self._data.pop(key)
- self._cache.pop(key, None)
def drop(self, key: str):
"""
@@ -418,8 +401,8 @@ class Flags:
"""
di = dios.DictOfSeries(columns=self.columns)
- for k, v in self._data.items():
- di[k] = self[k] # use cache
+ for k in self._data.keys():
+ di[k] = self[k]
return di.copy()
diff --git a/tests/core/test_flags.py b/tests/core/test_flags.py
index 742991b9e..b39ac4798 100644
--- a/tests/core/test_flags.py
+++ b/tests/core/test_flags.py
@@ -230,39 +230,6 @@ def test_set_flags_with_index(data: np.array):
flags[index, c] = wrong_len
-def test_cache():
- arr = np.array(
- [
- [0, 0, 0, 0],
- [0, 1, 2, 3],
- [0, 1, 2, 3],
- ]
- )
- data = pd.DataFrame(arr, dtype=float, columns=list("abcd"))
- flags = Flags(data)
-
- # cache empty
- assert flags._cache == {}
-
- # invoke caching
- flags["a"]
- assert "a" in flags._cache
-
- # clears cache
- flags["a"] = pd.Series([0, 0, 0], dtype=float)
- assert "a" not in flags._cache
-
- # cache all
- flags.toDios()
- for c in flags.columns:
- assert c in flags._cache
-
- # cache survive renaming
- flags.columns = list("xyzq")
- for c in flags.columns:
- assert c in flags._cache
-
-
def _validate_flags_equals_frame(flags, df):
assert df.columns.equals(flags.columns)
--
GitLab
From 8d2078d23033302ecea6bcb6fcda68b50be689b4 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Fri, 7 May 2021 11:43:18 +0200
Subject: [PATCH 160/180] [FIX] reduce runtime by ~45% (!) through the removal
of builtins.any
---
saqc/core/flags.py | 2 +-
saqc/core/history.py | 4 ++--
saqc/core/register.py | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index 63c48508b..cadfd090b 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -221,7 +221,7 @@ class Flags:
if colname:
errm += f"of column {colname} "
- if any(history.hist[0] != UNFLAGGED):
+ if (history.hist[0] != UNFLAGGED).any():
raise ValueError(errm + "missing an UNFLAGGED-column at first position")
# this ensures that the mask does not shadow UNFLAGGED with a NaN.
diff --git a/saqc/core/history.py b/saqc/core/history.py
index 23f660e34..cf219144f 100644
--- a/saqc/core/history.py
+++ b/saqc/core/history.py
@@ -439,7 +439,7 @@ class History:
f"'mask' must be of type pd.DataFrame, but {type(mask).__name__} was given"
)
- if any(mask.dtypes != bool):
+ if (mask.dtypes != bool).any():
raise ValueError("dtype of all columns in 'mask' must be bool")
if not mask.empty and not mask.iloc[:, -1].all():
@@ -467,7 +467,7 @@ class History:
f"'hist' must be of type pd.DataFrame, but {type(obj).__name__} was given"
)
- if any(obj.dtypes != float):
+ if (obj.dtypes != float).any():
raise ValueError("dtype of all columns in hist must be float")
if not obj.empty and (
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 80a06a0db..ac89eb6fb 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -243,7 +243,7 @@ def _maskData(
for c in columns:
col_mask = _isflagged(flags[c].to_numpy(), thresh)
- if any(col_mask):
+ if col_mask.any():
col_data = data[c].to_numpy(dtype=np.float64)
col_data[col_mask] = np.nan
@@ -357,7 +357,7 @@ def _unmaskData(data: dios.DictOfSeries, old_state: CallState) -> dios.DictOfSer
restore_old_mask = old_state.mask[c].to_numpy() & data[c].isna().to_numpy()
# we have nothing to restore
- if not any(restore_old_mask):
+ if not restore_old_mask.any():
continue
# restore old values if no new are present
--
GitLab
From 0ffd8c046927802d96f8c7e8761534f93d2d7e0f Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Fri, 7 May 2021 11:43:18 +0200
Subject: [PATCH 161/180] [FIX] reduce runtime by ~45% (!) through the removal
of builtins.any
---
saqc/core/history.py | 4 ++--
saqc/core/register.py | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/saqc/core/history.py b/saqc/core/history.py
index 79b78b579..85ae3aaea 100644
--- a/saqc/core/history.py
+++ b/saqc/core/history.py
@@ -427,7 +427,7 @@ class History:
f"'mask' must be of type pd.DataFrame, but {type(mask).__name__} was given"
)
- if any(mask.dtypes != bool):
+ if (mask.dtypes != bool).any():
raise ValueError("dtype of all columns in 'mask' must be bool")
if not mask.empty and not mask.iloc[:, -1].all():
@@ -455,7 +455,7 @@ class History:
f"'hist' must be of type pd.DataFrame, but {type(obj).__name__} was given"
)
- if any(obj.dtypes != float):
+ if (obj.dtypes != float).any():
raise ValueError("dtype of all columns in hist must be float")
if not obj.empty and (
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 80a06a0db..ac89eb6fb 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -243,7 +243,7 @@ def _maskData(
for c in columns:
col_mask = _isflagged(flags[c].to_numpy(), thresh)
- if any(col_mask):
+ if col_mask.any():
col_data = data[c].to_numpy(dtype=np.float64)
col_data[col_mask] = np.nan
@@ -357,7 +357,7 @@ def _unmaskData(data: dios.DictOfSeries, old_state: CallState) -> dios.DictOfSer
restore_old_mask = old_state.mask[c].to_numpy() & data[c].isna().to_numpy()
# we have nothing to restore
- if not any(restore_old_mask):
+ if not restore_old_mask.any():
continue
# restore old values if no new are present
--
GitLab
From 2f93f96ac4b9e03c54f3bd8a487c1d7335863bcb Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Fri, 7 May 2021 15:17:36 +0200
Subject: [PATCH 162/180] [HACK] track variable renames
---
saqc/core/translator/dmptranslator.py | 17 ++++++++++++++++-
tests/core/test_translator.py | 4 ++++
2 files changed, 20 insertions(+), 1 deletion(-)
diff --git a/saqc/core/translator/dmptranslator.py b/saqc/core/translator/dmptranslator.py
index f5397ab51..a420d1fb2 100644
--- a/saqc/core/translator/dmptranslator.py
+++ b/saqc/core/translator/dmptranslator.py
@@ -60,7 +60,22 @@ class DmpTranslator(Translator):
----
Could (and maybe should) be implemented as a method of `CallGraph`
"""
- return [SaQCFunction(name="")] + [f for l, f in call_stack if l.field == field]
+ out = [SaQCFunction(name="")]
+
+ for sel, func in call_stack:
+ if sel.field == field:
+ out.append(func)
+ # NOTE:
+ # This is an intermediary hack, to work around
+ # the problem, that field names are mutable and
+ # used as an mapping between `History` and
+ # `call_stack`. There are better ideas, to solve
+ # this (i.e. global function pointer) but for the
+ # moment this has to do the trick
+ if func.name == "tools.rename":
+ field = func.keywords.get("new_name") or func.args[3]
+
+ return out
def forward(self, flags: pd.DataFrame) -> Tuple[Flags, MaterializedGraph]:
"""
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index 4cca96a31..bffde47b1 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -298,3 +298,7 @@ def test_positionalMulitcallsPreserveState():
expected = tflags1[k].str.slice(start=1) * 2
got = tflags2[k].str.slice(start=1)
assert expected.equals(got)
+
+
+def test_dmpTranslatorRespectsRenames():
+ assert False
--
GitLab
From 186188c9db798c1aded6f80055312ea089d9cf30 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Mon, 10 May 2021 07:50:02 +0200
Subject: [PATCH 163/180] comments
---
saqc/core/translator/dmptranslator.py | 28 +++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/saqc/core/translator/dmptranslator.py b/saqc/core/translator/dmptranslator.py
index a420d1fb2..ce88d8690 100644
--- a/saqc/core/translator/dmptranslator.py
+++ b/saqc/core/translator/dmptranslator.py
@@ -159,6 +159,11 @@ class DmpTranslator(Translator):
flag_call_history = self._getFieldFunctions(field, call_graph)
flag_pos = flags.history[field].idxmax()
comments, causes = [], []
+ # NOTE:
+ # Strangely enough, this loop withstood all my efforts
+ # to speed it up through vectorization - the simple
+ # loop always outperformed even careful `pd.DataFrame.apply`
+ # versions. The latest try is left as a comment below.
for p in flag_pos:
func = flag_call_history[p]
cause = func.keywords.get("cause", self.ARGUMENTS["cause"])
@@ -180,3 +185,26 @@ class DmpTranslator(Translator):
}
out[field] = pd.DataFrame(var_flags)
return pd.concat(out, axis="columns")
+
+ # for field in tflags.columns:
+ # call_history = []
+ # for func in self._getFieldFunctions(field, call_graph):
+ # func_info = {
+ # "cause": func.keywords.get("cause", self.ARGUMENTS["comment"]),
+ # "comment": json.dumps({
+ # "test": func.name,
+ # "comment": func.keywords.get("comment", self.ARGUMENTS["comment"]),
+ # })
+ # }
+ # call_history.append(func_info)
+
+ # functions = pd.DataFrame(call_history)
+ # flag_pos = flags.history[field].idxmax()
+
+ # var_flags = {
+ # "quality_flag": tflags[field].reset_index(drop=True),
+ # "quality_comment": functions.loc[flag_pos, "comment"].reset_index(drop=True),
+ # "quality_cause": functions.loc[flag_pos, "cause"].reset_index(drop=True),
+ # }
+ # out[field] = pd.DataFrame(var_flags, index=flag_pos.index)
+ # return pd.concat(out, axis="columns")
--
GitLab
From f0267beedcd402621e9d1d88206374176ec48bc7 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Mon, 10 May 2021 07:50:25 +0200
Subject: [PATCH 164/180] flags with type 'category' to heavily reduce memorey
usage of saqc
---
saqc/core/history.py | 34 +++++++++++++++++++++-------------
1 file changed, 21 insertions(+), 13 deletions(-)
diff --git a/saqc/core/history.py b/saqc/core/history.py
index cf219144f..5b72a2641 100644
--- a/saqc/core/history.py
+++ b/saqc/core/history.py
@@ -78,7 +78,7 @@ class History:
hist = hist.copy()
mask = mask.copy()
- self.hist = hist
+ self.hist = hist.astype("category")
self.mask = mask
@property
@@ -166,7 +166,7 @@ class History:
touched = s.notna()
self.mask.iloc[touched, :pos] = False
- self.hist[pos] = s
+ self.hist[pos] = s.astype("category")
return self
@@ -269,7 +269,7 @@ class History:
# clear the current mask
self.mask.loc[(~value_mask & value_hist.notna()).any(axis="columns")] = False
- self.hist.loc[:, columns] = value_hist.copy()
+ self.hist.loc[:, columns] = value_hist.astype("category")
self.mask.loc[:, columns] = value_mask.copy()
return self
@@ -316,7 +316,7 @@ class History:
# the last column maybe is not entirely True, but
# the following append, will fix this
self.hist = self.hist.iloc[:, :-n]
- self.mask = self.mask.iloc[:, :-n]
+ self.mask = self.mask.iloc[:, :-n].astype("category")
self.append(s)
return self
@@ -383,10 +383,14 @@ class History:
-------
History
"""
- self.hist = self.hist.reindex(index=index, copy=False, fill_value=np.nan)
+ self.hist = self.hist.reindex(
+ index=index, copy=False, fill_value=np.nan
+ ).astype("category")
self.mask = self.mask.reindex(index=index, copy=False, fill_value=False)
# Note: all following code must handle empty frames
- self.hist.iloc[:, -1:] = self.hist.iloc[:, -1:].fillna(fill_value_last)
+ self.hist.iloc[:, -1:] = (
+ self.hist.iloc[:, -1:].fillna(fill_value_last).astype("category")
+ )
self.mask.iloc[:, -1:] = True
return self
@@ -467,8 +471,8 @@ class History:
f"'hist' must be of type pd.DataFrame, but {type(obj).__name__} was given"
)
- if (obj.dtypes != float).any():
- raise ValueError("dtype of all columns in hist must be float")
+ # if (obj.dtypes != float).any():
+ # raise ValueError("dtype of all columns in hist must be float")
if not obj.empty and (
not obj.columns.equals(pd.Index(range(len(obj.columns))))
@@ -490,8 +494,8 @@ class History:
f"value must be of type pd.Series, but {type(obj).__name__} was given"
)
- if not obj.dtype == float:
- raise ValueError("dtype must be float")
+ # if not obj.dtype == float:
+ # raise ValueError("dtype must be float")
return obj
@@ -545,12 +549,16 @@ def applyFunctionOnHistory(
new_history = History()
if func_handle_df:
- history.hist = hist_func(history.hist, **hist_kws)
+ history.hist = hist_func(history.hist.astype(float), **hist_kws).astype(
+ "category"
+ )
history.mask = hist_func(history.mask, **mask_kws)
else:
for pos in history.columns:
- new_history.hist[pos] = hist_func(history.hist[pos], **hist_kws)
+ new_history.hist[pos] = hist_func(
+ history.hist[pos].astype(float), **hist_kws
+ ).astype("category")
new_history.mask[pos] = mask_func(history.mask[pos], **mask_kws)
# handle unstable state
@@ -560,7 +568,7 @@ def applyFunctionOnHistory(
if isinstance(last_column, str) and last_column == "dummy":
last_column = pd.Series(UNTOUCHED, index=new_history.index, dtype=float)
- new_history.append(last_column, force=True)
+ new_history.append(last_column.astype("category"), force=True)
# assure a boolean mask and UNFLAGGED column
new_history.mask = new_history.mask.fillna(True).astype(bool)
--
GitLab
From 0618f5eedf38a0b4866997eb071f58bd06941135 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Mon, 10 May 2021 18:21:13 +0200
Subject: [PATCH 165/180] trying to get the categoricals consistent (it's not
going to be easy!)
---
saqc/core/flags.py | 3 ---
saqc/core/history.py | 34 ++++++++++++++++++++--------------
tests/core/test_history.py | 4 +++-
3 files changed, 23 insertions(+), 18 deletions(-)
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index cadfd090b..76bdd9318 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -1,7 +1,5 @@
#!/usr/bin/env python
-
from __future__ import annotations
-from dios.dios import DictOfSeries
import pandas as pd
import dios
@@ -10,7 +8,6 @@ from typing import Mapping, Union, Dict, DefaultDict, Optional, Type, Tuple, Ite
from saqc.constants import *
from saqc.core.history import History
-from saqc.lib.types import PandasLike
_VAL = Union[pd.Series, History]
DictLike = Union[
diff --git a/saqc/core/history.py b/saqc/core/history.py
index 5b72a2641..7fcf47300 100644
--- a/saqc/core/history.py
+++ b/saqc/core/history.py
@@ -315,8 +315,8 @@ class History:
# this may leave us in an unstable state, because
# the last column maybe is not entirely True, but
# the following append, will fix this
- self.hist = self.hist.iloc[:, :-n]
- self.mask = self.mask.iloc[:, :-n].astype("category")
+ self.hist = self.hist.iloc[:, :-n].astype("category")
+ self.mask = self.mask.iloc[:, :-n]
self.append(s)
return self
@@ -329,7 +329,7 @@ class History:
-------
pd.Series: maximum values
"""
- return self.hist[self.mask].idxmax(axis=1)
+ return self.hist[self.mask].astype(float).idxmax(axis=1)
def max(self) -> pd.Series:
"""
@@ -383,15 +383,18 @@ class History:
-------
History
"""
- self.hist = self.hist.reindex(
+ hist = self.hist.astype(float).reindex(
index=index, copy=False, fill_value=np.nan
- ).astype("category")
- self.mask = self.mask.reindex(index=index, copy=False, fill_value=False)
- # Note: all following code must handle empty frames
- self.hist.iloc[:, -1:] = (
- self.hist.iloc[:, -1:].fillna(fill_value_last).astype("category")
)
- self.mask.iloc[:, -1:] = True
+ mask = self.mask.astype(bool).reindex(index=index, copy=False, fill_value=False)
+
+ # Note: all following code must handle empty frames
+ hist.iloc[:, -1:] = hist.iloc[:, -1:].fillna(fill_value_last)
+ mask.iloc[:, -1:] = True
+
+ self.mask = mask.astype(bool)
+ self.hist.hist = hist.astype("category")
+
return self
def __copy__(self, deep: bool = True):
@@ -471,8 +474,10 @@ class History:
f"'hist' must be of type pd.DataFrame, but {type(obj).__name__} was given"
)
- # if (obj.dtypes != float).any():
- # raise ValueError("dtype of all columns in hist must be float")
+ if obj.dtypes.isin([float, pd.Categorical]).any() is False:
+ raise ValueError(
+ "dtype of all columns in hist must be float or categorical"
+ )
if not obj.empty and (
not obj.columns.equals(pd.Index(range(len(obj.columns))))
@@ -494,8 +499,8 @@ class History:
f"value must be of type pd.Series, but {type(obj).__name__} was given"
)
- # if not obj.dtype == float:
- # raise ValueError("dtype must be float")
+ if not ((obj.dtype == float) or isinstance(obj.dtype, pd.CategoricalDtype)):
+ raise ValueError("dtype must be float or categorical")
return obj
@@ -549,6 +554,7 @@ def applyFunctionOnHistory(
new_history = History()
if func_handle_df:
+ # we need to pass the data as floats as functions may fail with Categorical
history.hist = hist_func(history.hist.astype(float), **hist_kws).astype(
"category"
)
diff --git a/tests/core/test_history.py b/tests/core/test_history.py
index ebeab59ba..c3c32a909 100644
--- a/tests/core/test_history.py
+++ b/tests/core/test_history.py
@@ -87,7 +87,9 @@ def check_invariants(hist):
assert isinstance(hist, History)
assert isinstance(hist.hist, pd.DataFrame)
assert isinstance(hist.mask, pd.DataFrame)
- assert all(hist.hist.dtypes == float)
+ assert all(
+ [isinstance(dtype, (float, pd.CategoricalDtype)) for dtype in hist.hist.dtypes]
+ )
assert all(hist.mask.dtypes == bool)
assert hist.hist.columns.equals(hist.mask.columns)
assert hist.columns is hist.hist.columns
--
GitLab
From 81cd2779f2d68849c42d00d349df1456091cd3d7 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Mon, 10 May 2021 20:23:16 +0200
Subject: [PATCH 166/180] backtrack variable naming changes through
tools.rename
---
saqc/core/lib.py | 5 ++++-
saqc/core/translator/dmptranslator.py | 23 +++++++++++++-------
saqc/lib/types.py | 2 +-
tests/core/test_translator.py | 31 +++++++++++++++++++++++++--
4 files changed, 49 insertions(+), 12 deletions(-)
diff --git a/saqc/core/lib.py b/saqc/core/lib.py
index e487f5bc3..ce19f2458 100644
--- a/saqc/core/lib.py
+++ b/saqc/core/lib.py
@@ -58,7 +58,10 @@ class SaQCFunction:
return self.name
def __repr__(self):
- return f"{self.__class__.__name__}.{self.func.__name__}"
+ args = ", ".join(self.args)
+ kwargs = ", ".join([f"{k}={v}" for k, v in self.keywords.items()])
+ string = ",".join(filter(None, [args, kwargs]))
+ return f"{self.__class__.__name__}.{self.func.__name__}({string})"
def bind(self, *args, **keywords):
return SaQCFunction(
diff --git a/saqc/core/translator/dmptranslator.py b/saqc/core/translator/dmptranslator.py
index ce88d8690..8d95efd3c 100644
--- a/saqc/core/translator/dmptranslator.py
+++ b/saqc/core/translator/dmptranslator.py
@@ -59,19 +59,26 @@ class DmpTranslator(Translator):
Note
----
Could (and maybe should) be implemented as a method of `CallGraph`
+
+ Currently we work around the issue, that we keep track of the
+ computations we do on a variable using the variable name, but also
+ allow mutations of that name (i.e. our key) through `tools.rename`
+ in a somewhat hacky way. There are better ideas, to solve this (i.e.
+ global function pointers), but for the moment this has to do the trick
"""
- out = [SaQCFunction(name="")]
+ # backtrack name changes and let's look, if our field
+ # originally had another name
+ for sel, func in call_stack[::-1]:
+ if func.name == "tools.rename":
+ new_name = func.keywords.get("new_name") or func.args[3]
+ if new_name == field:
+ field = sel.field
+ out = [SaQCFunction(name="")]
for sel, func in call_stack:
if sel.field == field:
out.append(func)
- # NOTE:
- # This is an intermediary hack, to work around
- # the problem, that field names are mutable and
- # used as an mapping between `History` and
- # `call_stack`. There are better ideas, to solve
- # this (i.e. global function pointer) but for the
- # moment this has to do the trick
+ # forward track name changes
if func.name == "tools.rename":
field = func.keywords.get("new_name") or func.args[3]
diff --git a/saqc/lib/types.py b/saqc/lib/types.py
index d13d01274..f4cfa2633 100644
--- a/saqc/lib/types.py
+++ b/saqc/lib/types.py
@@ -39,7 +39,7 @@ FreqString = NewType(
)
CallGraph = List[Tuple[ColumnSelector, APIController, SaQCFunction]]
-MaterializedGraph = List[Tuple[ColumnSelector, Optional[SaQCFunction]]]
+MaterializedGraph = List[Tuple[ColumnSelector, SaQCFunction]]
# we define a bunch of type aliases, mostly needed to generate appropiate fuzzy data through hypothesis
ColumnName = NewType("ColumnName", str)
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index bffde47b1..56ef3f217 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -12,6 +12,7 @@ import pytest
from dios import DictOfSeries
from saqc.constants import UNFLAGGED, BAD, DOUBTFUL
+from saqc.core import translator
from saqc.core.translator import (
FloatTranslator,
PositionalTranslator,
@@ -300,5 +301,31 @@ def test_positionalMulitcallsPreserveState():
assert expected.equals(got)
-def test_dmpTranslatorRespectsRenames():
- assert False
+def test_smpTranslatorHandlesRenames():
+
+ data = initData(3)
+
+ this: str = data.columns[0]
+ other: str = this + "_new"
+
+ saqc = (
+ SaQC(data=data)
+ .outliers.flagRange(this, min=1, max=10)
+ .tools.rename(this, other)
+ .breaks.flagMissing(other, min=4, max=6)
+ )
+ saqc = saqc.evaluate()
+
+ this_funcs = DmpTranslator._getFieldFunctions(this, saqc._computed)
+ other_funcs = DmpTranslator._getFieldFunctions(other, saqc._computed)
+
+ assert [f.name for f in this_funcs] == [
+ "",
+ "outliers.flagRange",
+ "tools.rename",
+ "breaks.flagMissing",
+ ]
+
+ # we skip the first function in both lists, as they are dummy functions
+ # inserted to allow a proper replay of all function calls
+ assert this_funcs[1:] == other_funcs[1:]
--
GitLab
From b52ff7b44f32733f295820af3cd9332737c91cc9 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Tue, 11 May 2021 09:01:49 +0200
Subject: [PATCH 167/180] DmpTranslator: enforce valid quality_cause
---
saqc/core/translator/dmptranslator.py | 23 ++++++++++++++-
tests/core/test_translator.py | 41 +++++++++++++++++++++++++--
2 files changed, 60 insertions(+), 4 deletions(-)
diff --git a/saqc/core/translator/dmptranslator.py b/saqc/core/translator/dmptranslator.py
index ce88d8690..5b04d1076 100644
--- a/saqc/core/translator/dmptranslator.py
+++ b/saqc/core/translator/dmptranslator.py
@@ -22,6 +22,7 @@ from saqc.lib.types import MaterializedGraph
from saqc.core.translator.basetranslator import Translator, ForwardMap
+
class DmpTranslator(Translator):
"""
@@ -38,6 +39,18 @@ class DmpTranslator(Translator):
"BAD": BAD,
}
+ _QUALITY_CAUSES = {
+ "BATTERY_LOW",
+ "BELOW_MINIMUM",
+ "ABOVE_MAXIMUM",
+ "BELOW_OR_ABOVE_MIN_MAX",
+ "ISOLATED_SPIKE",
+ "DEFECTIVE_SENSOR",
+ "LEFT_CENSORED_DATA",
+ "RIGHT_CENSORED_DATA",
+ "OTHER",
+ }
+
def __init__(self):
super().__init__(forward=self._FORWARD)
@@ -178,10 +191,18 @@ class DmpTranslator(Translator):
causes.append(cause)
comments.append(comment)
+ # DMP quality_cause needs some special care as only certain values
+ # and combinations are allowed.
+ # See: https://wiki.intranet.ufz.de/wiki/dmp/index.php/Qualit%C3%A4tsflags
+ causes = pd.Series(causes, index=flags[field].index)
+ causes[(causes == self.ARGUMENTS["cause"]) & (flags[field] > GOOD)] = "OTHER"
+ if not ((causes == "") | causes.isin(self._QUALITY_CAUSES)).all():
+ raise ValueError(f"quality causes needs to be one of {self._QUALITY_CAUSES}")
+
var_flags = {
"quality_flag": tflags[field],
"quality_comment": pd.Series(comments, index=flags[field].index),
- "quality_cause": pd.Series(causes, index=flags[field].index),
+ "quality_cause": causes,
}
out[field] = pd.DataFrame(var_flags)
return pd.concat(out, axis="columns")
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index bffde47b1..da3223351 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -117,17 +117,36 @@ def test_dmpTranslator():
tflags.loc[:, ("var1", "quality_comment")]
== '{"test": "flagBar", "comment": "I did it"}'
).all(axis=None)
+ assert (
+ tflags.loc[:, ("var1", "quality_cause")]
+ == "OTHER"
+ ).all(axis=None)
+
assert (tflags.loc[:, ("var2", "quality_flag")] == "BAD").all(axis=None)
assert (
tflags.loc[:, ("var2", "quality_comment")]
== '{"test": "flagFoo", "comment": ""}'
).all(axis=None)
+ assert (
+ tflags.loc[:, ("var2", "quality_cause")]
+ == "OTHER"
+ ).all(axis=None)
assert (
tflags.loc[flags["var3"] == BAD, ("var3", "quality_comment")]
== '{"test": "flagInit", "comment": "initial flags"}'
).all(axis=None)
+ assert (
+ tflags.loc[flags["var3"] == BAD, ("var3", "quality_cause")]
+ == "OTHER"
+ ).all(axis=None)
+ assert (
+ tflags.loc[flags["var3"] < DOUBTFUL, ("var3", "quality_cause")]
+ == ""
+ ).all(axis=None)
+
+
def test_positionalTranslator():
@@ -167,12 +186,12 @@ def test_positionalTranslatorIntegration():
def test_dmpTranslatorIntegration():
- data = initData(3)
+ data = initData(1)
col = data.columns[0]
translator = DmpTranslator()
saqc = SaQC(data=data, translator=translator)
- saqc = saqc.breaks.flagMissing(col).outliers.flagRange(col, min=3, max=10)
+ saqc = saqc.outliers.flagRange(col, min=3, max=10)
data, flags = saqc.getResult()
qflags = flags.xs("quality_flag", axis="columns", level=1)
@@ -183,7 +202,7 @@ def test_dmpTranslatorIntegration():
assert qflags.isin(translator._forward.keys()).all(axis=None)
assert qfunc.isin({"", "breaks.flagMissing", "outliers.flagRange"}).all(axis=None)
- assert (qcause == "").all(axis=None)
+ assert (qcause[qflags[col] == "BAD"] == "OTHER").all(axis=None)
round_trip = translator.backward(*translator.forward(flags))
@@ -197,6 +216,22 @@ def test_dmpTranslatorIntegration():
flags.xs("quality_cause", axis="columns", level=1)
)
+def test_dmpValidCause():
+ data = initData(1)
+ col = data.columns[0]
+
+ translator = DmpTranslator()
+ saqc = SaQC(data=data, translator=translator)
+ saqc = saqc.outliers.flagRange(col, min=3, max=10, cause="SOMETHING_STUPID")
+ with pytest.raises(ValueError):
+ data, flags = saqc.getResult()
+
+ saqc = saqc.outliers.flagRange(col, min=3, max=10, cause="BELOW_OR_ABOVE_MIN_MAX")
+ data, flags = saqc.getResult()
+ qflags = flags.xs("quality_flag", axis="columns", level=1)
+ qcause = flags.xs("quality_cause", axis="columns", level=1)
+ assert (qcause[qflags[col] == "BAD"] == "BELOW_OR_ABOVE_MIN_MAX").all(axis=None)
+ assert (qcause[qflags[col] != "BAD"] == "").all(axis=None)
def _buildupSaQCObjects():
--
GitLab
From 758b26a330989256eb75b801f16a5e6d508f543f Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Wed, 12 May 2021 23:19:11 +0200
Subject: [PATCH 168/180] [FIX] reduce the memory consumption of SaQC by >50%
through Histories of type pd.Categorical
---
saqc/core/history.py | 2 +-
tests/core/test_history.py | 5 ++++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/saqc/core/history.py b/saqc/core/history.py
index 7fcf47300..5be12bd61 100644
--- a/saqc/core/history.py
+++ b/saqc/core/history.py
@@ -78,7 +78,7 @@ class History:
hist = hist.copy()
mask = mask.copy()
- self.hist = hist.astype("category")
+ self.hist = hist.astype("category", copy=copy)
self.mask = mask
@property
diff --git a/tests/core/test_history.py b/tests/core/test_history.py
index c3c32a909..1dd94b71e 100644
--- a/tests/core/test_history.py
+++ b/tests/core/test_history.py
@@ -188,7 +188,10 @@ def test_copy(data):
assert deep.hist is not hist.hist
assert deep.mask is not hist.mask
- assert shallow.hist is hist.hist
+ # we need to convert to and from categoricals in order
+ # to allow all operations on `History`, that way we loose
+ # the identity
+ # assert shallow.hist is hist.hist
assert shallow.mask is hist.mask
--
GitLab
From d2ae8d76599fa76631691c5dcfacf4d239e18ff1 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Wed, 12 May 2021 23:20:00 +0200
Subject: [PATCH 169/180] changed SaQC parameter name
---
saqc/__main__.py | 2 +-
saqc/core/core.py | 11 ++++----
saqc/core/translator/dmptranslator.py | 9 ++++---
tests/core/test_translator.py | 37 ++++++++++-----------------
4 files changed, 27 insertions(+), 32 deletions(-)
diff --git a/saqc/__main__.py b/saqc/__main__.py
index d26f9d3c3..14b6386d3 100644
--- a/saqc/__main__.py
+++ b/saqc/__main__.py
@@ -111,7 +111,7 @@ def main(config, data, scheme, outfile, nodata, log_level, fail):
saqc = SaQC(
data=data,
nodata=nodata,
- translator=SCHEMES[scheme or "float"](),
+ scheme=SCHEMES[scheme or "float"](),
error_policy="raise" if fail else "warn",
)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 64d9b1faf..55f3a5821 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -9,12 +9,13 @@ from __future__ import annotations
import logging
import copy as stdcopy
from typing import Tuple, Sequence, Union, Optional
-from dios import DictOfSeries, to_dios
from typing_extensions import Literal
+import inspect
import pandas as pd
import numpy as np
-import inspect
+
+from dios import DictOfSeries, to_dios
from saqc.constants import *
from saqc.core.flags import initFlagsLike, Flags
@@ -123,7 +124,7 @@ class SaQC(FuncModules):
self,
data,
flags=None,
- translator: Translator = None,
+ scheme: Translator = None,
nodata=np.nan,
error_policy="raise",
):
@@ -133,7 +134,7 @@ class SaQC(FuncModules):
self._nodata = nodata
self._flags = self._initFlags(data, flags)
self._error_policy = error_policy
- self._translator = translator or FloatTranslator()
+ self._translator = scheme or FloatTranslator()
# NOTE:
# We need two lists to represent the future and the past computations
@@ -182,7 +183,7 @@ class SaQC(FuncModules):
flags=Flags(),
nodata=self._nodata,
error_policy=self._error_policy,
- translator=self._translator,
+ scheme=self._translator,
)
for k, v in injectables.items():
if not hasattr(out, k):
diff --git a/saqc/core/translator/dmptranslator.py b/saqc/core/translator/dmptranslator.py
index 7c12244c9..e0847413a 100644
--- a/saqc/core/translator/dmptranslator.py
+++ b/saqc/core/translator/dmptranslator.py
@@ -22,7 +22,6 @@ from saqc.lib.types import MaterializedGraph
from saqc.core.translator.basetranslator import Translator, ForwardMap
-
class DmpTranslator(Translator):
"""
@@ -202,9 +201,13 @@ class DmpTranslator(Translator):
# and combinations are allowed.
# See: https://wiki.intranet.ufz.de/wiki/dmp/index.php/Qualit%C3%A4tsflags
causes = pd.Series(causes, index=flags[field].index)
- causes[(causes == self.ARGUMENTS["cause"]) & (flags[field] > GOOD)] = "OTHER"
+ causes[
+ (causes == self.ARGUMENTS["cause"]) & (flags[field] > GOOD)
+ ] = "OTHER"
if not ((causes == "") | causes.isin(self._QUALITY_CAUSES)).all():
- raise ValueError(f"quality causes needs to be one of {self._QUALITY_CAUSES}")
+ raise ValueError(
+ f"quality causes needs to be one of {self._QUALITY_CAUSES}"
+ )
var_flags = {
"quality_flag": tflags[field],
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index 450103da3..ccd9de77b 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -118,36 +118,25 @@ def test_dmpTranslator():
tflags.loc[:, ("var1", "quality_comment")]
== '{"test": "flagBar", "comment": "I did it"}'
).all(axis=None)
- assert (
- tflags.loc[:, ("var1", "quality_cause")]
- == "OTHER"
- ).all(axis=None)
-
+ assert (tflags.loc[:, ("var1", "quality_cause")] == "OTHER").all(axis=None)
assert (tflags.loc[:, ("var2", "quality_flag")] == "BAD").all(axis=None)
assert (
tflags.loc[:, ("var2", "quality_comment")]
== '{"test": "flagFoo", "comment": ""}'
).all(axis=None)
- assert (
- tflags.loc[:, ("var2", "quality_cause")]
- == "OTHER"
- ).all(axis=None)
+ assert (tflags.loc[:, ("var2", "quality_cause")] == "OTHER").all(axis=None)
assert (
tflags.loc[flags["var3"] == BAD, ("var3", "quality_comment")]
== '{"test": "flagInit", "comment": "initial flags"}'
).all(axis=None)
- assert (
- tflags.loc[flags["var3"] == BAD, ("var3", "quality_cause")]
- == "OTHER"
- ).all(axis=None)
- assert (
- tflags.loc[flags["var3"] < DOUBTFUL, ("var3", "quality_cause")]
- == ""
- ).all(axis=None)
-
-
+ assert (tflags.loc[flags["var3"] == BAD, ("var3", "quality_cause")] == "OTHER").all(
+ axis=None
+ )
+ assert (tflags.loc[flags["var3"] < DOUBTFUL, ("var3", "quality_cause")] == "").all(
+ axis=None
+ )
def test_positionalTranslator():
@@ -169,7 +158,7 @@ def test_positionalTranslatorIntegration():
col: str = data.columns[0]
translator = PositionalTranslator()
- saqc = SaQC(data=data, translator=translator)
+ saqc = SaQC(data=data, scheme=translator)
saqc = saqc.breaks.flagMissing(col).outliers.flagRange(
col, min=3, max=10, flag=DOUBTFUL
)
@@ -191,8 +180,8 @@ def test_dmpTranslatorIntegration():
col = data.columns[0]
translator = DmpTranslator()
- saqc = SaQC(data=data, translator=translator)
- saqc = saqc.outliers.flagRange(col, min=3, max=10)
+ saqc = SaQC(data=data, scheme=translator)
+ saqc = saqc.breaks.flagMissing(col).outliers.flagRange(col, min=3, max=10)
data, flags = saqc.getResult()
qflags = flags.xs("quality_flag", axis="columns", level=1)
@@ -217,12 +206,13 @@ def test_dmpTranslatorIntegration():
flags.xs("quality_cause", axis="columns", level=1)
)
+
def test_dmpValidCause():
data = initData(1)
col = data.columns[0]
translator = DmpTranslator()
- saqc = SaQC(data=data, translator=translator)
+ saqc = SaQC(data=data, scheme=translator)
saqc = saqc.outliers.flagRange(col, min=3, max=10, cause="SOMETHING_STUPID")
with pytest.raises(ValueError):
data, flags = saqc.getResult()
@@ -234,6 +224,7 @@ def test_dmpValidCause():
assert (qcause[qflags[col] == "BAD"] == "BELOW_OR_ABOVE_MIN_MAX").all(axis=None)
assert (qcause[qflags[col] != "BAD"] == "").all(axis=None)
+
def _buildupSaQCObjects():
"""
--
GitLab
From 30ba092af1544281993eeb6c68f2888237b099e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?David=20Sch=C3=A4fer?= <david.schaefer@ufz.de>
Date: Thu, 20 May 2021 19:37:07 +0000
Subject: [PATCH 170/180] Apply 3 suggestion(s) to 1 file(s)
---
saqc/core/core.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 55f3a5821..eee5bf915 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -17,7 +17,6 @@ import numpy as np
from dios import DictOfSeries, to_dios
-from saqc.constants import *
from saqc.core.flags import initFlagsLike, Flags
from saqc.core.lib import APIController, ColumnSelector
from saqc.core.register import FUNC_MAP, SaQCFunction
@@ -141,10 +140,10 @@ class SaQC(FuncModules):
# on a `SaQC`-Object. Due to the dynamic nature of field expansion
# with regular expressions, we can't just reuse the original execution
# plan to infer all translation related information.
- self._planned: CallGraph = [] # will be filled by calls to `_wrap`
+ self._planned: CallGraph = [] # will be filled by calls to `_wrap`
self._computed: MaterializedGraph = self._translator.buildGraph(
self._flags
- ) # will be filled in `evaluate`
+ ) # will be filled in `evaluate`
@staticmethod
def _initFlags(data: DictOfSeries, flags: Optional[Flags]) -> Flags:
--
GitLab
From 27470a4f595d002371f174b32fdff8114fdf66da Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Fri, 21 May 2021 07:44:47 +0200
Subject: [PATCH 171/180] deleted dangling file
---
saqc/core/translator.py | 243 ----------------------------------------
1 file changed, 243 deletions(-)
delete mode 100644 saqc/core/translator.py
diff --git a/saqc/core/translator.py b/saqc/core/translator.py
deleted file mode 100644
index 05c73bc8b..000000000
--- a/saqc/core/translator.py
+++ /dev/null
@@ -1,243 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-# TODO:
-# - remove `MaterializedGraph` from `Translator.forward`
-# - own directory
-# - parameter checking
-# - fix exe
-from __future__ import annotations
-
-from typing import Dict, Optional, Union, Any, Tuple, Callable
-
-import numpy as np
-import pandas as pd
-
-from dios import DictOfSeries
-
-from saqc.core.lib import SaQCFunction, ColumnSelector
-from saqc.core.flags import (
- Flags,
- UNFLAGGED,
- GOOD,
- DOUBTFUL,
- BAD,
-)
-from saqc.lib.types import ExternalFlag, MaterializedGraph
-
-
-ForwardMap = Dict[ExternalFlag, float]
-BackwardMap = Dict[float, ExternalFlag]
-
-
-class Translator:
- """
- This class provides the basic translation mechanism and should serve as
- a base class for every other translation scheme.
-
- The general translation is realized through dictionary lookups, altough
- we might need to extend this logic to also allow calls to translation
- functions in the future. Currently at least one `dict` defining the
- 'forward' translation from 'user flags' -> 'internal flags' needs to be
- provided.
- Optionally a second `dict` can be passed to map 'internal flags' -> 'user flags',
- if the latter is not given, this 'backwards' translation will inferred as
- the inverse of the 'forward' translation.
-
- The translation mechanism imposes a few restrictions:
- - The scheme must be well definied, i.e. we need a backward translation for
- every forward translation (each value in `self._forward` needs a key in
- `self._backward`).
- - We need translations for the special flags `saqc.constants.UNFLAGGED` and
- `saqc.constants.BAD`. That implies, that every valid translation scheme
- provides at least one user flag that maps to `BAD` and one that maps to
- `UNFLAGGED`.
- """
-
- # (internal) threshold flag above which values will be masked
- TO_MASK: Union[float, bool] = True
-
- # additional arguments and default values the translation scheme accepts
- ARGUMENTS: Dict[str, Any] = {}
-
- def __init__(self, forward: ForwardMap, backward: Optional[BackwardMap] = None):
- """
- Parameters
- ----------
- forward : dict
- A mapping defining the forward translation of scalar flag values
-
- backward : dict, optinal
- A mapping defining the backward translation of scalar flag values.
- If not given, `backward` is inferred from `forward`
-
- Note
- ----
- `backward` needs to provide a mappinf for the two special flags
- `saqc.core.UNFLAGGED`, `saqc.core.BAD`
- """
- # NOTE: we also add the keys to also allow the usage of internal flags
- self._forward = forward
- if backward is None:
- backward = {v: k for k, v in forward.items()}
- if {UNFLAGGED, BAD} - set(backward.keys()):
- raise ValueError(
- f"need translations for the special flags `UNFLAGGED` ({UNFLAGGED}) and `BAD` ({BAD})"
- )
- self._backward = backward
-
- @staticmethod
- def _translate(
- flags: Union[Flags, pd.DataFrame, pd.Series],
- trans_map: Union[ForwardMap, BackwardMap],
- ) -> DictOfSeries:
- """
- Translate a given flag data structure to another according to the
- mapping given in `trans_map`
-
- Parameters
- ----------
- flags : Flags, pd.DataFrame
- The flags to translate
-
- Returns
- -------
- pd.DataFrame, Flags
- """
- if isinstance(flags, pd.Series):
- flags = flags.to_frame()
-
- out = DictOfSeries()
- expected = pd.Index(trans_map.values())
- for field in flags.columns:
- out[field] = flags[field].replace(trans_map)
- diff = pd.Index(out[field]).difference(expected)
- if not diff.empty:
- raise ValueError(
- f"flags were not translated: {diff.drop_duplicates().to_list()}"
- )
- return out
-
- def __call__(self, flag: ExternalFlag) -> float:
- """
- Translate a scalar 'external flag' to an 'internal flag'
-
- Parameters
- ----------
- flag : float, int, str
- The external flag to translate
-
- Returns
- -------
- float
- """
- if flag not in self._forward:
- if flag not in self._backward:
- raise ValueError(f"invalid flag: {flag}")
- return flag # type: ignore -> if flag is in `self._backward` it is of type float
- return self._forward[flag]
-
- @staticmethod
- def _generateInitFunction(
- flag_name: str, history: pd.Series
- ) -> Callable[[DictOfSeries, str, Flags, Any], Tuple[DictOfSeries, Flags]]:
- # NOTE:
- # Close over `flags_column` and `history_column`
- # to immitate the original function application,
- # that we cannot replicate directly because of
- # lacking information.
- # I am not entirely sure, if closing over
- # `flag_column` is really necessary or if we
- # even should close over `flags`
- def mapFlags(data: DictOfSeries, field: str, flags: Flags, **kwargs):
- flags[history.index, flag_name] = history
- return data, flags
-
- return mapFlags
-
- @staticmethod
- def _buildCallGraph(flags: Flags) -> MaterializedGraph:
- """
- build a call graph from the `Flags` and their `History`
-
- As we usually don't have enough information (i.e. SaQC
- function name and all used parameters) we generate dummy
- functions here. These dummy functions unconditionally set
- the `field` to the provided flags.
-
- The idea is, to spit out an `MaterializedGraph`, that can
- be used in replays of the original `SaQC` run in gives the
- same result for the same input data set.
-
- Parameters
- ----------
- flags : flags to generate a call graph for
- """
- out = []
- for flag_name in flags.columns:
- # skip the default column
- for _, hist_column in tuple(flags.history[flag_name].hist.items())[1:]:
- out.append(
- (
- ColumnSelector(flag_name),
- SaQCFunction(
- name="initFlags",
- function=Translator._generateInitFunction(
- flag_name, hist_column
- ),
- ),
- )
- )
- return out
-
- def forward(self, flags: pd.DataFrame) -> Tuple[Flags, MaterializedGraph]:
- """
- Translate from 'external flags' to 'internal flags'
-
- Parameters
- ----------
- flags : pd.DataFrame
- The external flags to translate
-
- Returns
- -------
- Flags object
- """
- tflags = Flags(self._translate(flags, self._forward))
- graph = self._buildCallGraph(tflags)
- return tflags, graph
-
- def backward(self, flags: Flags, call_graph: MaterializedGraph) -> pd.DataFrame:
- """
- Translate from 'internal flags' to 'external flags'
-
- Parameters
- ----------
- flags : pd.DataFrame
- The external flags to translate
- call_stack : List
- The saqc functions called to generate the given `flags` (i.e. `SaQC._computed`)
- `call_stack` is not evaluated here, it's presence only ensures, that subclasses
- have access to it.
-
- Returns
- -------
- pd.DataFrame
- """
- return self._translate(flags, self._backward).to_df()
-
-
-class FloatTranslator(Translator):
-
- """
- Acts as the default Translator, provides a changeable subset of the
- internal float flags
- """
-
- _FORWARD: Dict[float, float] = {
- -np.inf: -np.inf,
- **{k: k for k in np.arange(0, 256, dtype=float)},
- }
-
- def __init__(self):
- super().__init__(self._FORWARD)
--
GitLab
From ac883c3244887f0c49c7d259fa02a79362e5a2a0 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Tue, 25 May 2021 08:18:46 +0200
Subject: [PATCH 172/180] review requests
---
saqc/core/core.py | 7 ++++---
saqc/lib/types.py | 4 ++--
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index eee5bf915..d060e8acd 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -242,6 +242,10 @@ class SaQC(FuncModules):
data_result, flags_result = _saqcCallFunc(
sel, control, func, data, flags
)
+ # we check the passed function-kwargs after the actual call,
+ # because now "hard" errors would already have been raised
+ # (eg. `TypeError: got multiple values for argument 'data'`,
+ # when the user pass data=...)
_warnForUnusedKwargs(function, self._translator)
computed.append((sel, func))
except Exception as e:
@@ -356,9 +360,6 @@ def _saqcCallFunc(locator, controller, function, data, flags):
data_result, flags_result = function(data, field, flags)
- # we check the passed function-kwargs after the actual call, because now "hard" errors would already have been
- # raised (Eg. `TypeError: got multiple values for argument 'data'`, when the user pass data=...)
-
return data_result, flags_result
diff --git a/saqc/lib/types.py b/saqc/lib/types.py
index f4cfa2633..dad548f90 100644
--- a/saqc/lib/types.py
+++ b/saqc/lib/types.py
@@ -11,10 +11,10 @@ __all__ = [
"TimestampColumnName",
"CurveFitter",
"ExternalFlag",
- "CallStack",
- "CalledStack",
"PositiveFloat",
"PositiveInt",
+ "CallGraph",
+ "MaterializedGraph",
]
from typing import TypeVar, Union, NewType, List, Tuple, Optional
--
GitLab
From bc93021461e14909c4ca92a6b0a86340b3d1c34b Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 26 May 2021 15:10:56 +0200
Subject: [PATCH 173/180] minor change in a comment
---
saqc/core/translator/basetranslator.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/saqc/core/translator/basetranslator.py b/saqc/core/translator/basetranslator.py
index d70c88bc2..1461cfb61 100644
--- a/saqc/core/translator/basetranslator.py
+++ b/saqc/core/translator/basetranslator.py
@@ -127,7 +127,7 @@ class Translator:
if flag not in self._forward:
if flag not in self._backward:
raise ValueError(f"invalid flag: {flag}")
- return flag # type: ignore -> if flag is in `self._backward` it is of type float
+ return flag # type: # ignore -> if flag is in `self._backward` it is of type float
return self._forward[flag]
@staticmethod
--
GitLab
From f31ba14ffa625f28a4d2162aa2e31f6d0f340604 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Wed, 26 May 2021 16:03:54 +0200
Subject: [PATCH 174/180] blackified again
---
saqc/core/translator/basetranslator.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/saqc/core/translator/basetranslator.py b/saqc/core/translator/basetranslator.py
index 1461cfb61..c13456269 100644
--- a/saqc/core/translator/basetranslator.py
+++ b/saqc/core/translator/basetranslator.py
@@ -127,7 +127,9 @@ class Translator:
if flag not in self._forward:
if flag not in self._backward:
raise ValueError(f"invalid flag: {flag}")
- return flag # type: # ignore -> if flag is in `self._backward` it is of type float
+ return (
+ flag
+ ) # type: # ignore -> if flag is in `self._backward` it is of type float
return self._forward[flag]
@staticmethod
--
GitLab
From 23c1f0931b35b40f7ea363ec27d585018831a7af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?David=20Sch=C3=A4fer?= <david.schaefer@ufz.de>
Date: Thu, 27 May 2021 11:10:15 +0000
Subject: [PATCH 175/180] Apply 1 suggestion(s) to 1 file(s)
---
saqc/core/translator/dmptranslator.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/saqc/core/translator/dmptranslator.py b/saqc/core/translator/dmptranslator.py
index e0847413a..6bbbc743d 100644
--- a/saqc/core/translator/dmptranslator.py
+++ b/saqc/core/translator/dmptranslator.py
@@ -48,6 +48,7 @@ class DmpTranslator(Translator):
"LEFT_CENSORED_DATA",
"RIGHT_CENSORED_DATA",
"OTHER",
+ "AUTO_FLAGGED",
}
def __init__(self):
--
GitLab
From 1a1cf9cd01b75d6bb034257c8d8e5791cc2de414 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 27 May 2021 14:21:07 +0200
Subject: [PATCH 176/180] non optional `backward` parameter
---
saqc/core/translator/basetranslator.py | 22 +++++++++-------------
saqc/core/translator/dmptranslator.py | 4 +++-
tests/core/test_translator.py | 4 ++--
3 files changed, 14 insertions(+), 16 deletions(-)
diff --git a/saqc/core/translator/basetranslator.py b/saqc/core/translator/basetranslator.py
index c13456269..033849ddf 100644
--- a/saqc/core/translator/basetranslator.py
+++ b/saqc/core/translator/basetranslator.py
@@ -53,30 +53,26 @@ class Translator:
# additional arguments and default values the translation scheme accepts
ARGUMENTS: Dict[str, Any] = {}
- def __init__(self, forward: ForwardMap, backward: Optional[BackwardMap] = None):
+ def __init__(self, forward: ForwardMap, backward: BackwardMap):
"""
Parameters
----------
forward : dict
A mapping defining the forward translation of scalar flag values
- backward : dict, optinal
- A mapping defining the backward translation of scalar flag values.
- If not given, `backward` is inferred from `forward`
+ backward : dict
+ A mapping defining the backward translation of scalar flag values
Note
----
- `backward` needs to provide a mappinf for the two special flags
- `saqc.core.UNFLAGGED`, `saqc.core.BAD`
+ `backward` needs to provide a mapping for the two special flags
+ `saqc.constants.UNFLAGGED`, `saqc.constants.BAD`
"""
- # NOTE: we also add the keys to also allow the usage of internal flags
- self._forward = forward
- if backward is None:
- backward = {v: k for k, v in forward.items()}
- if {UNFLAGGED, BAD} - set(backward.keys()):
+ if UNFLAGGED not in backward or BAD not in backward:
raise ValueError(
f"need translations for the special flags `UNFLAGGED` ({UNFLAGGED}) and `BAD` ({BAD})"
)
+ self._forward = forward
self._backward = backward
@staticmethod
@@ -228,10 +224,10 @@ class FloatTranslator(Translator):
internal float flags
"""
- _FORWARD: ForwardMap = {
+ _MAP = {
-np.inf: -np.inf,
**{k: k for k in np.arange(0, 256, dtype=float)},
}
def __init__(self):
- super().__init__(self._FORWARD)
+ super().__init__(self._MAP, self._MAP)
diff --git a/saqc/core/translator/dmptranslator.py b/saqc/core/translator/dmptranslator.py
index 6bbbc743d..a53ceeb12 100644
--- a/saqc/core/translator/dmptranslator.py
+++ b/saqc/core/translator/dmptranslator.py
@@ -52,7 +52,9 @@ class DmpTranslator(Translator):
}
def __init__(self):
- super().__init__(forward=self._FORWARD)
+ super().__init__(
+ forward=self._FORWARD, backward={v: k for k, v in self._FORWARD.items()}
+ )
@staticmethod
def _getFieldFunctions(
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index ccd9de77b..6761f1d52 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -34,7 +34,7 @@ def _genTranslators():
dtype(-1): BAD,
**{dtype(f * 10): float(f) for f in range(10)},
}
- translator = Translator(flags)
+ translator = Translator(flags, {v: k for k, v in flags.items()})
yield flags, translator
@@ -327,7 +327,7 @@ def test_positionalMulitcallsPreserveState():
assert expected.equals(got)
-def test_smpTranslatorHandlesRenames():
+def test_dmpTranslatorHandlesRenames():
data = initData(3)
--
GitLab
From d4435d6cbfea596ffd9053dae3a5f1d22b0fcf1f Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 27 May 2021 14:53:06 +0200
Subject: [PATCH 177/180] deactivate DmpTranslator for now
---
saqc/core/translator/dmptranslator.py | 1 +
tests/core/test_translator.py | 6 +++++-
tests/integration/test_integration.py | 3 ++-
3 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/saqc/core/translator/dmptranslator.py b/saqc/core/translator/dmptranslator.py
index a53ceeb12..781c7c6a5 100644
--- a/saqc/core/translator/dmptranslator.py
+++ b/saqc/core/translator/dmptranslator.py
@@ -52,6 +52,7 @@ class DmpTranslator(Translator):
}
def __init__(self):
+ raise NotImplementedError
super().__init__(
forward=self._FORWARD, backward={v: k for k, v in self._FORWARD.items()}
)
diff --git a/tests/core/test_translator.py b/tests/core/test_translator.py
index 6761f1d52..b294b3ca9 100644
--- a/tests/core/test_translator.py
+++ b/tests/core/test_translator.py
@@ -82,6 +82,7 @@ def test_backwardTranslationFail():
translator.backward(flags, None)
+@pytest.mark.skip(reason="dmp translator implementation is currently blocked")
def test_dmpTranslator():
translator = DmpTranslator()
@@ -174,6 +175,7 @@ def test_positionalTranslatorIntegration():
assert (flags.columns == round_trip.columns).all()
+@pytest.mark.skip(reason="dmp translator implementation is currently blocked")
def test_dmpTranslatorIntegration():
data = initData(1)
@@ -207,6 +209,7 @@ def test_dmpTranslatorIntegration():
)
+@pytest.mark.skip(reason="dmp translator implementation is currently blocked")
def test_dmpValidCause():
data = initData(1)
col = data.columns[0]
@@ -327,7 +330,8 @@ def test_positionalMulitcallsPreserveState():
assert expected.equals(got)
-def test_dmpTranslatorHandlesRenames():
+@pytest.mark.skip(reason="dmp translator implementation is currently blocked")
+def test_smpTranslatorHandlesRenames():
data = initData(3)
diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py
index c4e0bb046..595b92a3f 100644
--- a/tests/integration/test_integration.py
+++ b/tests/integration/test_integration.py
@@ -20,6 +20,7 @@ def test__main__py():
result = runner.invoke(saqc.__main__.main, args)
assert result.exit_code == 0, result.output
- for scheme in ["float", "dmp", "positional"]:
+ # for scheme in ["float", "dmp", "positional"]:
+ for scheme in ["float", "positional"]:
result = runner.invoke(saqc.__main__.main, args + ["--scheme", scheme])
assert result.exit_code == 0, result.output
--
GitLab
From 095f1f3765ffc47877d25804fe0b8290ed8a7739 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 27 May 2021 15:01:46 +0200
Subject: [PATCH 178/180] remove comment
---
saqc/core/core.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index d060e8acd..876568302 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -269,8 +269,6 @@ class SaQC(FuncModules):
if any([control.plot for _, control, _ in self._planned]):
plotAllHook(data, flags)
- # This is way faster for big datasets, than to throw everything in the constructor.
- # Simply because of _initFlags -> merge() -> mergeDios() over all columns.
return self._construct(
_flags=flags, _data=data, _computed=self._computed + computed
)
--
GitLab
From 8890f25b18ce8b38ad3734fb5ddafad20c673592 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 27 May 2021 15:28:46 +0200
Subject: [PATCH 179/180] static expansion of regular expression fields
---
saqc/core/core.py | 103 ++++++++-----------
saqc/core/lib.py | 3 +-
saqc/core/reader.py | 29 +++---
saqc/core/visitor.py | 2 +-
tests/core/test_reader.py | 5 +-
tests/funcs/test_generic_config_functions.py | 2 +-
6 files changed, 61 insertions(+), 83 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 876568302..b04823f2a 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -8,6 +8,7 @@ from __future__ import annotations
import logging
import copy as stdcopy
+from saqc.lib.tools import toSequence
from typing import Tuple, Sequence, Union, Optional
from typing_extensions import Literal
import inspect
@@ -194,26 +195,7 @@ class SaQC(FuncModules):
from saqc.core.reader import readConfig
out = stdcopy.deepcopy(self)
- out._planned.extend(readConfig(fname, self._flags, self._nodata))
- return out
-
- @staticmethod
- def _expandFields(
- selector: ColumnSelector, func: SaQCFunction, variables: pd.Index
- ) -> Sequence[Tuple[ColumnSelector, SaQCFunction]]:
- if not selector.regex:
- return [(selector, func)]
-
- out = []
- for field in variables[variables.str.match(selector.field)]:
- out.append(
- (
- ColumnSelector(
- field=field, target=selector.target, regex=selector.regex
- ),
- func,
- )
- )
+ out._planned.extend(readConfig(fname, self._data, self._nodata))
return out
def evaluate(self):
@@ -233,38 +215,34 @@ class SaQC(FuncModules):
data, flags = self._data, self._flags
computed: MaterializedGraph = []
for selector, control, function in self._planned:
- for sel, func in self._expandFields(
- selector, function, data.columns.union(flags.columns)
- ):
- logger.debug(f"processing: {sel.field}, {func.name}, {func.keywords}")
-
- try:
- data_result, flags_result = _saqcCallFunc(
- sel, control, func, data, flags
- )
- # we check the passed function-kwargs after the actual call,
- # because now "hard" errors would already have been raised
- # (eg. `TypeError: got multiple values for argument 'data'`,
- # when the user pass data=...)
- _warnForUnusedKwargs(function, self._translator)
- computed.append((sel, func))
- except Exception as e:
- _handleErrors(e, sel.field, control, func, self._error_policy)
- continue
-
- if control.plot:
- plotHook(
- data_old=data,
- data_new=data_result,
- flagger_old=flags,
- flagger_new=flags_result,
- sources=[],
- targets=[sel.field],
- plot_name=func.name,
- )
+ logger.debug(f"processing: {selector.field}, {function.name}, {function.keywords}")
+ try:
+ data_result, flags_result = _saqcCallFunc(
+ selector, control, function, data, flags
+ )
+ # we check the passed function-kwargs after the actual call,
+ # because now "hard" errors would already have been raised
+ # (eg. `TypeError: got multiple values for argument 'data'`,
+ # when the user pass data=...)
+ _warnForUnusedKwargs(function, self._translator)
+ computed.append((selector, function))
+ except Exception as e:
+ _handleErrors(e, selector.field, control, function, self._error_policy)
+ continue
+
+ if control.plot:
+ plotHook(
+ data_old=data,
+ data_new=data_result,
+ flagger_old=flags,
+ flagger_new=flags_result,
+ sources=[],
+ targets=[selector.field],
+ plot_name=function.name,
+ )
- data = data_result
- flags = flags_result
+ data = data_result
+ flags = flags_result
if any([control.plot for _, control, _ in self._planned]):
plotAllHook(data, flags)
@@ -304,23 +282,26 @@ class SaQC(FuncModules):
**fkwargs,
) -> SaQC:
- fkwargs.setdefault("to_mask", self._translator.TO_MASK)
+ if regex and target is not None:
+ raise ValueError("explicit `target` not supported with `regex=True`")
- control = APIController(plot=plot)
+ fkwargs.setdefault("to_mask", self._translator.TO_MASK)
- locator = ColumnSelector(
- field=field,
- target=target if target is not None else field,
- regex=regex,
- )
+ out = self if inplace else self.copy(deep=True)
+ control = APIController(plot=plot)
partial = func.bind(
*fargs,
**{"nodata": self._nodata, "flag": self._translator(flag), **fkwargs},
)
- out = self if inplace else self.copy(deep=True)
- out._planned.append((locator, control, partial))
+ fields = self._data.columns.str.match(field) if regex else toSequence(field)
+ for field in fields:
+ locator = ColumnSelector(
+ field=field,
+ target=target if target is not None else field,
+ )
+ out._planned.append((locator, control, partial))
return out
@@ -352,7 +333,7 @@ def _saqcCallFunc(locator, controller, function, data, flags):
field = locator.field
target = locator.target
- if (target != field) and (locator.regex is False):
+ if target != field:
data, flags = copy(data, field, flags, target)
field = target
diff --git a/saqc/core/lib.py b/saqc/core/lib.py
index ce19f2458..575308a9e 100644
--- a/saqc/core/lib.py
+++ b/saqc/core/lib.py
@@ -9,10 +9,9 @@ from typing_extensions import Literal
class ColumnSelector:
- def __init__(self, field, target=None, regex=False):
+ def __init__(self, field, target=None):
self.field = field
self.target = target or field
- self.regex = regex
def __repr__(self):
return f"{self.__class__.__name__}({self.field})"
diff --git a/saqc/core/reader.py b/saqc/core/reader.py
index d13ec81a2..7e2b82a26 100644
--- a/saqc/core/reader.py
+++ b/saqc/core/reader.py
@@ -10,7 +10,7 @@ from saqc.core.config import Fields as F
from saqc.core.visitor import ConfigFunctionParser
from saqc.core.lib import ConfigController
from saqc.core.register import FUNC_MAP
-from saqc.lib.tools import isQuoted
+from saqc.lib.tools import isQuoted, toSequence
COMMENT = "#"
@@ -56,36 +56,37 @@ def _injectOptionalColumns(df):
return df
-def _parseConfig(df, flags, nodata):
+def _parseConfig(df, data, nodata):
funcs = []
for lineno, (_, target, expr, plot) in enumerate(df.itertuples()):
if target == "None" or pd.isnull(target) or pd.isnull(expr):
continue
- regex = False
if isQuoted(target):
- regex = True
target = target[1:-1]
+ target = data.columns[data.columns.str.match(target)]
tree = ast.parse(expr, mode="eval")
- func_name, kwargs = ConfigFunctionParser(flags).parse(tree.body)
+ func_name, kwargs = ConfigFunctionParser().parse(tree.body)
func = FUNC_MAP[func_name]
- selector = ColumnSelector(
- field=kwargs.get("field", target),
- target=target,
- regex=regex,
- )
-
control = ConfigController(plot=plot, lineno=lineno + 2, expression=expr)
f = func.bind(**{"nodata": nodata, **kwargs})
- funcs.append((selector, control, f))
+ targets = toSequence(target)
+
+ for target in targets:
+ selector = ColumnSelector(
+ field=kwargs.get("field", target),
+ target=target,
+ )
+ funcs.append((selector, control, f))
+
return funcs
-def readConfig(fname, flags, nodata):
+def readConfig(fname, data, nodata):
df = pd.read_csv(
fname,
sep=r"\s*;\s*",
@@ -104,4 +105,4 @@ def readConfig(fname, flags, nodata):
df[F.TEST] = df[F.TEST].replace(r"^\s*$", np.nan, regex=True)
df[F.PLOT] = df[F.PLOT].replace({"False": "", EMPTY: "", np.nan: ""})
df = df.astype({F.PLOT: bool})
- return _parseConfig(df, flags, nodata)
+ return _parseConfig(df, data, nodata)
diff --git a/saqc/core/visitor.py b/saqc/core/visitor.py
index d301d3108..8e07ddc6d 100644
--- a/saqc/core/visitor.py
+++ b/saqc/core/visitor.py
@@ -137,7 +137,7 @@ class ConfigFunctionParser(ast.NodeVisitor):
ast.Attribute,
)
- def __init__(self, flags):
+ def __init__(self):
self.kwargs = {}
self.environment = {
diff --git a/tests/core/test_reader.py b/tests/core/test_reader.py
index 3efec94fe..0cb25f8b4 100644
--- a/tests/core/test_reader.py
+++ b/tests/core/test_reader.py
@@ -49,10 +49,7 @@ def test_variableRegex(data):
for regex, expected in tests:
fobj = writeIO(header + "\n" + f"{regex} ; flagtools.flagDummy()")
saqc = SaQC(data).readConfig(fobj)
- expansion = saqc._expandFields(
- saqc._planned[0][0], saqc._planned[0][2], data.columns
- )
- result = [s.field for s, _ in expansion]
+ result = [s.field for s, _, _ in saqc._planned]
assert np.all(result == expected)
diff --git a/tests/funcs/test_generic_config_functions.py b/tests/funcs/test_generic_config_functions.py
index 78658d646..1c560db44 100644
--- a/tests/funcs/test_generic_config_functions.py
+++ b/tests/funcs/test_generic_config_functions.py
@@ -40,7 +40,7 @@ def data_diff():
def _compileGeneric(expr, flags):
tree = ast.parse(expr, mode="eval")
- _, kwargs = ConfigFunctionParser(flags).parse(tree.body)
+ _, kwargs = ConfigFunctionParser().parse(tree.body)
return kwargs["func"]
--
GitLab
From b280b40299bb46b8fd955cf8bbe9f997bac9506a Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Thu, 27 May 2021 16:03:01 +0200
Subject: [PATCH 180/180] blackblackblack is sooo not white
---
saqc/core/core.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index b04823f2a..19a631270 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -215,7 +215,9 @@ class SaQC(FuncModules):
data, flags = self._data, self._flags
computed: MaterializedGraph = []
for selector, control, function in self._planned:
- logger.debug(f"processing: {selector.field}, {function.name}, {function.keywords}")
+ logger.debug(
+ f"processing: {selector.field}, {function.name}, {function.keywords}"
+ )
try:
data_result, flags_result = _saqcCallFunc(
selector, control, function, data, flags
--
GitLab