diff --git a/saqc/core/core.py b/saqc/core/core.py
index 8041f6f4f1fea1cf66f03ff6d197961bf43fdd26..03cb065778552f57b04a23278a2c19f40d2bfcf2 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -4,7 +4,7 @@ from __future__ import annotations
# TODO:
# - integrate plotting into the api
-# - `data` and `flagger` as arguments to `getResult`
+# - `data` and `flags` as arguments to `getResult`
import logging
import copy as stdcopy
@@ -114,15 +114,16 @@ class SaQC(FuncModules):
self._to_call: List[Tuple[ColumnSelector, APIController, SaQCFunction]] = []
def _initFlags(self, data, flags: Union[Flags, None]):
- """ Init the internal flagger object.
+ """ Init the internal Flags-object.
- Ensures that all data columns are present and user passed flags from
- a flags frame or an already initialised flagger are used.
+ Ensures that all data columns are present and user passed
+ flags from a frame or an already initialised Flags-object
+ are used.
"""
if flags is None:
return initFlagsLike(data)
- # add columns that are present in data but not in flagger
+ # add columns that are present in data but not in flags
for c in data.columns.difference(flags.columns):
flags[c] = pd.Series(UNFLAGGED, index=data[c].index, dtype=float)
@@ -202,7 +203,7 @@ class SaQC(FuncModules):
plotAllHook(data, flags)
# This is way faster for big datasets, than to throw everything in the constructor.
- # Simply because of _initFlagger -> merge() -> mergeDios() over all columns.
+ # Simply because of _initFlags -> merge() -> mergeDios() over all columns.
new = self._constructSimple()
new._flags, new._data = flags, data
return new
@@ -213,7 +214,7 @@ class SaQC(FuncModules):
Returns
-------
- data, flagger: (DictOfSeries, DictOfSeries)
+ data, flags: (DictOfSeries, DictOfSeries)
"""
realization = self.evaluate()
@@ -270,7 +271,7 @@ class SaQC(FuncModules):
def _saqcCallFunc(locator, controller, function, data, flags):
# NOTE:
# We assure that all columns in data have an equivalent column in flags,
- # we might have more flagger columns though
+ # we might have more flags columns though
assert data.columns.difference(flags.columns).empty
field = locator.field
diff --git a/saqc/core/flags.py b/saqc/core/flags.py
index 1d95adf310ebac9e5ecaf89030578471061e64f4..d698cf1d0ae1cddc641af64e01069e71b35afc43 100644
--- a/saqc/core/flags.py
+++ b/saqc/core/flags.py
@@ -202,7 +202,7 @@ class Flags:
# technically it would be possible to select a field and set
# the entire column to a scalar flag value (float), but it has
# a high potential, that this is not intended by the user.
- # if desired use ``flagger[:, field] = flag``
+ # if desired use ``flags[:, field] = flag``
if not isinstance(value, pd.Series):
raise ValueError("must pass value of type pd.Series")
diff --git a/saqc/core/register.py b/saqc/core/register.py
index b00f353d08953a0b728a6b85f02d504093049b02..256b3228a07dae46815445f161cbc3e38a74197e 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -131,14 +131,14 @@ def _postCall(result, old_state: CallState) -> FuncReturnT:
Parameters
----------
result : tuple
- the result from the called function, namely: data and flagger
+ the result from the called function, namely: data and flags
old_state : dict
control keywords from `_preCall`
Returns
-------
- data, flagger : dios.DictOfSeries, saqc.flagger.Flagger
+ data, flags : dios.DictOfSeries, saqc.Flags
"""
data, flags = result
flags = _restoreFlags(flags, old_state)
@@ -264,7 +264,7 @@ def _prepareFlags(flags: Flags, masking) -> Flags:
Prepare flags before each call. Always returns a copy.
Currently this only clears the flags, but in future,
- this should be sliced the flagger to the columns, that
+ this should be sliced the flags to the columns, that
the saqc-function needs.
"""
# Either the index or the columns itself changed
@@ -286,9 +286,9 @@ def _restoreFlags(flags: Flags, old_state: CallState):
out = old_state.flags.copy()
for c in columns:
- # this implicitly squash the new-flagger history (RHS) to a single column, which than is appended to
- # the old history (LHS). The new-flagger history possibly consist of multiple columns, one for each
- # time flags was set to the flagger.
+ # this implicitly squash the new flags history (RHS) to a single column, which than is appended to
+ # the old history (LHS). The new flags history possibly consist of multiple columns, one for each
+ # time a series or scalar was passed to the flags.
out[c] = flags[c]
return out
diff --git a/saqc/funcs/breaks.py b/saqc/funcs/breaks.py
index f3ad9eeb2e8c7ae84c832a056725b883009c163c..b6dd00834fda7e82be409fac24706f61e4d8f0a0 100644
--- a/saqc/funcs/breaks.py
+++ b/saqc/funcs/breaks.py
@@ -41,8 +41,8 @@ def flagMissing(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
nodata : any, default np.nan
A value that defines missing data.
flag : float, default BAD
@@ -52,9 +52,8 @@ def flagMissing(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
"""
datacol = data[field]
if np.isnan(nodata):
@@ -88,8 +87,8 @@ def flagIsolated(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object
+ flags : saqc.Flags
+ A flags object
gap_window : str
The minimum size of the gap before and after a group of valid values, making this group considered an
isolated group. See condition (2) and (3)
@@ -103,8 +102,8 @@ def flagIsolated(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional information related to `data`.
+ flags : saqc.Flags
+ The flags object, holding flags and additional information related to `data`.
Notes
-----
@@ -160,8 +159,8 @@ def flagJumps(
A dictionary of pandas.Series, holding all the data.
field : str
The reference variable, the deviation from wich determines the flagging.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
thresh : float
The threshold, the mean of the values have to change by, to trigger flagging.
winsz : str
diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py
index 83439157ec0687f6210a3c9f69962835ea649518..450a77602c8af5bd6dbc584a7d7cc4f7288a1a18 100644
--- a/saqc/funcs/changepoints.py
+++ b/saqc/funcs/changepoints.py
@@ -47,8 +47,8 @@ def flagChangePoints(
A dictionary of pandas.Series, holding all the data.
field : str
The reference variable, the deviation from wich determines the flagging.
- flagger : saqc.flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.flags
+ A flags object, holding flags and additional informations related to `data`.
stat_func : Callable[numpy.array, numpy.array]
A function that assigns a value to every twin window. Left window content will be passed to first variable,
right window content will be passed to the second.
@@ -140,8 +140,8 @@ def assignChangePointCluster(
A dictionary of pandas.Series, holding all the data.
field : str
The reference variable, the deviation from wich determines the flagging.
- flagger : saqc.flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.flags
+ A flags object, holding flags and additional informations related to `data`.
stat_func : Callable[[numpy.array, numpy.array], float]
A function that assigns a value to every twin window. Left window content will be passed to first variable,
right window content will be passed to the second.
diff --git a/saqc/funcs/constants.py b/saqc/funcs/constants.py
index 3791eaabfe164ada318ef713210c8b6dfead6e68..6f8d29828a505ecc2fb33d57e8a8654f54be88e3 100644
--- a/saqc/funcs/constants.py
+++ b/saqc/funcs/constants.py
@@ -43,8 +43,8 @@ def flagConstants(
A dictionary of pandas.Series, holding all the data.
field : str
Name of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
thresh : float
Upper bound for the maximum total change of an interval to be flagged constant.
window : str
@@ -56,9 +56,9 @@ def flagConstants(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional informations related to `data`.
- Flags values may have changed, relatively to the flagger input.
+ flags : saqc.Flags
+ The flags object, holding flags and additional informations related to `data`.
+ Flags values may have changed, relatively to the flags input.
"""
if not isinstance(window, str):
raise TypeError('window must be offset string.')
@@ -104,8 +104,8 @@ def flagByVariance(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
window : str
Only intervals of minimum size "window" have the chance to get flagged as constant intervals
thresh : float
@@ -125,9 +125,9 @@ def flagByVariance(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional informations related to `data`.
- Flags values may have changed, relatively to the flagger input.
+ flags : saqc.Flags
+ The flags object, holding flags and additional informations related to `data`.
+ Flags values may have changed, relatively to the flags input.
"""
dataseries = data[field]
diff --git a/saqc/funcs/curvefit.py b/saqc/funcs/curvefit.py
index 3465e07d141d62d84855469161f69e8ebea951d2..4b50693c6ed31f33b34d39d0224e6f5390a69b09 100644
--- a/saqc/funcs/curvefit.py
+++ b/saqc/funcs/curvefit.py
@@ -74,8 +74,8 @@ def fitPolynomial(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-modelled.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
winsz : {str, int}
The size of the window you want to use for fitting. If an integer is passed, the size
refers to the number of periods for every fitting window. If an offset string is passed,
@@ -106,9 +106,8 @@ def fitPolynomial(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
"""
# TODO: some (rater large) parts are functional similar to saqc.funcs.rolling.roll
if data[field].empty:
diff --git a/saqc/funcs/drift.py b/saqc/funcs/drift.py
index 65f5b043dfd22ee3a6327cff7bc935d15a501940..f47183c6fc29dd21accb0aada073f267fc645271 100644
--- a/saqc/funcs/drift.py
+++ b/saqc/funcs/drift.py
@@ -55,8 +55,8 @@ def flagDriftFromNorm(
A dictionary of pandas.Series, holding all the data.
field : str
A dummy parameter.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
fields : str
List of fieldnames in data, determining which variables are to be included into the flagging process.
segment_freq : str
@@ -86,9 +86,9 @@ def flagDriftFromNorm(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the input flagger.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the input flags.
Notes
-----
@@ -166,8 +166,8 @@ def flagDriftFromReference(
A dictionary of pandas.Series, holding all the data.
field : str
The reference variable, the deviation from wich determines the flagging.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
fields : str
List of fieldnames in data, determining wich variables are to be included into the flagging process.
segment_freq : str
@@ -186,9 +186,9 @@ def flagDriftFromReference(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the input flagger.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the input flags.
Notes
-----
@@ -255,8 +255,8 @@ def flagDriftFromScaledNorm(
A dictionary of pandas.Series, holding all the data.
field : str
A dummy parameter.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
fields_scale1 : str
List of fieldnames in data to be included into the flagging process which are scaled according to scaling
scheme 1.
@@ -290,9 +290,9 @@ def flagDriftFromScaledNorm(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the input flagger.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the input flags.
References
----------
@@ -386,8 +386,8 @@ def correctExponentialDrift(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to correct.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
maint_data_field : str
The fieldname of the datacolumn holding the maintenance information.
The maint data is to expected to have following form:
@@ -406,9 +406,8 @@ def correctExponentialDrift(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
"""
# 1: extract fit intervals:
if data[maint_data_field].empty:
@@ -480,8 +479,8 @@ def correctRegimeAnomaly(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to correct.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
cluster_field : str
A string denoting the field in data, holding the cluster label for the data you want to correct.
model : Callable
@@ -502,8 +501,8 @@ def correctRegimeAnomaly(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ The quality flags of data
"""
cluster_ser = data[cluster_field]
unique_successive = pd.unique(cluster_ser.values)
@@ -583,8 +582,8 @@ def correctOffset(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to correct.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
max_mean_jump : float
when searching for changepoints in mean - this is the threshold a mean difference in the
sliding window search must exceed to trigger changepoint detection.
@@ -606,8 +605,8 @@ def correctOffset(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ The quality flags of data
"""
data, flags = copy(data, field, flags, field + '_CPcluster')
data, flags = assignChangePointCluster(
@@ -689,8 +688,8 @@ def flagRegimeAnomaly(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
cluster_field : str
The name of the column in data, holding the cluster labels for the samples in field. (has to be indexed
equal to field)
@@ -711,9 +710,9 @@ def flagRegimeAnomaly(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional informations related to `data`.
- Flags values may have changed, relatively to the flagger input.
+ flags : saqc.Flags
+ The flags object, holding flags and additional informations related to `data`.
+ Flags values may have changed, relatively to the flags input.
"""
return assignRegimeAnomaly(
data, field, flags,
@@ -767,8 +766,8 @@ def assignRegimeAnomaly(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
cluster_field : str
The name of the column in data, holding the cluster labels for the samples in field. (has to be indexed
equal to field)
@@ -794,9 +793,9 @@ def assignRegimeAnomaly(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional informations related to `data`.
- Flags values may have changed, relatively to the flagger input.
+ flags : saqc.Flags
+ The flags object, holding flags and additional informations related to `data`.
+ Flags values may have changed, relatively to the flags input.
"""
series = data[cluster_field]
cluster = np.unique(series)
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index 94b04da10df0c1ff1f94973c0314a95947a9537c..5cb90778114ee5211da928068478f4fe4ddba4bb 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -24,8 +24,8 @@ def forceFlags(
data container
field : str
columns name that holds the data
- flagger : Flagger
- flagger object
+ flags : saqc.Flags
+ flags object
flag : float, default BAD
flag to set
kwargs : dict
@@ -34,7 +34,7 @@ def forceFlags(
Returns
-------
data : DictOfSeries
- flagger : Flagger
+ flags : saqc.Flags
See Also
--------
@@ -57,15 +57,15 @@ def clearFlags(data: DictOfSeries, field: ColumnName, flags: Flags, **kwargs) ->
data container
field : str
columns name that holds the data
- flagger : Flagger
- flagger object
+ flags : saqc.Flags
+ flags object
kwargs : dict
unused
Returns
-------
data : DictOfSeries
- flagger : Flagger
+ flags : saqc.Flags
See Also
--------
@@ -93,8 +93,8 @@ def flagUnflagged(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
flag : float, default BAD
flag value to set
kwargs : Dict
@@ -104,8 +104,8 @@ def flagUnflagged(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ The quality flags of data
See Also
--------
@@ -128,15 +128,15 @@ def flagGood(data: DictOfSeries, field: ColumnName, flags: Flags, flag=BAD, **kw
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
Returns
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ The quality flags of data
"""
warnings.warn("'flagGood' is deprecated and does nothing, use 'flagUnflagged' instead", DeprecationWarning)
return data, flags
@@ -165,8 +165,8 @@ def flagManual(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
mdata : {pd.Series, pd.Dataframe, DictOfSeries}
The "manually generated" data
mflag : scalar
@@ -189,7 +189,7 @@ def flagManual(
Returns
-------
data : original data
- flagger : modified flagger
+ flags : modified flags
Examples
--------
@@ -204,7 +204,7 @@ def flagManual(
On *dayly* data, with the 'ontime' method, only the provided timestamnps are used.
Bear in mind that only exact timestamps apply, any offset will result in ignoring
the timestamp.
- >>> _, fl = flagManual(data, field, flagger, mdata, mflag=1, method='ontime')
+ >>> _, fl = flagManual(data, field, flags, mdata, mflag=1, method='ontime')
>>> fl[field] > UNFLAGGED
2000-01-31 False
2000-02-01 True
@@ -217,7 +217,7 @@ def flagManual(
Freq: D, dtype: bool
With the 'right-open' method, the mdata is forward fill:
- >>> _, fl = flagManual(data, field, flagger, mdata, mflag=1, method='right-open')
+ >>> _, fl = flagManual(data, field, flags, mdata, mflag=1, method='right-open')
>>> fl[field] > UNFLAGGED
2000-01-31 False
2000-02-01 True
@@ -229,7 +229,7 @@ def flagManual(
Freq: D, dtype: bool
With the 'left-open' method, backward filling is used:
- >>> _, fl = flagManual(data, field, flagger, mdata, mflag=1, method='left-open')
+ >>> _, fl = flagManual(data, field, flags, mdata, mflag=1, method='left-open')
>>> fl[field] > UNFLAGGED
2000-01-31 False
2000-02-01 False
@@ -287,7 +287,7 @@ def flagManual(
@register(masking='none', module="flagtools")
def flagDummy(data: DictOfSeries, field: ColumnName, flags: Flags, **kwargs) -> Tuple[DictOfSeries, Flags]:
"""
- Function does nothing but returning data and flagger.
+ Function does nothing but returning data and flags.
Parameters
----------
@@ -295,15 +295,15 @@ def flagDummy(data: DictOfSeries, field: ColumnName, flags: Flags, **kwargs) ->
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
Returns
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ The quality flags of data
"""
return data, flags
@@ -319,8 +319,8 @@ def flagForceFail(data: DictOfSeries, field: ColumnName, flags: Flags, **kwargs)
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
Raises
------
diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py
index 329514fcf511a0d95866950e1522b1e46a26a605..93a7eec9bba2dd637f52b0c6c1e9a987bbda40f7 100644
--- a/saqc/funcs/generic.py
+++ b/saqc/funcs/generic.py
@@ -108,8 +108,8 @@ def process(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, where you want the result from the generic expressions processing to be written to.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
func : Callable
The data processing function with parameter names that will be
interpreted as data column entries.
@@ -122,9 +122,9 @@ def process(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
The shape of the data may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- The flags shape may have changed relatively to the input flagger.
+ flags : saqc.Flags
+ The quality flags of data
+ The flags shape may have changed relatively to the input flags.
Examples
--------
@@ -184,8 +184,8 @@ def flag(
field : str
The fieldname of the column, where you want the result from the generic expressions evaluation to be projected
to.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
func : Callable
The expression that is to be evaluated is passed in form of a callable, with parameter names that will be
interpreted as data column entries. The Callable must return an boolen array like.
@@ -199,9 +199,9 @@ def flag(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the flags input.
Examples
--------
@@ -247,9 +247,9 @@ def flag(
if field not in flags.columns:
flags[field] = pd.Series(UNFLAGGED, index=mask.index, name=field)
- # if flagger.getFlags(field).empty:
- # flagger = flagger.merge(
- # flagger.initFlags(
+ # if flags.getFlags(field).empty:
+ # flags = flags.merge(
+ # flags.initFlags(
# data=pd.Series(name=field, index=mask.index, dtype=np.float64)))
flags[mask, field] = flag
return data, flags
diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py
index 32fcd2ba45568d1a1e8fd4234f19ba484649b2d5..1c5131257856835e20e9e54a3f42a7e44ea92ee2 100644
--- a/saqc/funcs/interpolation.py
+++ b/saqc/funcs/interpolation.py
@@ -39,8 +39,8 @@ def interpolateByRolling(
field : str
Name of the column, holding the data-to-be-interpolated.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Information related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional Information related to `data`.
winsz : int, str
The size of the window, the aggregation is computed from. An integer define the number of periods to be used,
@@ -65,9 +65,8 @@ def interpolateByRolling(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
"""
data = data.copy()
@@ -118,8 +117,8 @@ def interpolateInvalid(
field : str
Name of the column, holding the data-to-be-interpolated.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Information related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional Information related to `data`.
method : {"linear", "time", "nearest", "zero", "slinear", "quadratic", "cubic", "spline", "barycentric",
"polynomial", "krogh", "piecewise_polynomial", "spline", "pchip", "akima"}
@@ -146,9 +145,8 @@ def interpolateInvalid(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
"""
inter_data = interpolateNANs(
data[field],
@@ -203,8 +201,8 @@ def interpolateIndex(
field : str
Name of the column, holding the data-to-be-interpolated.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Information related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional Information related to `data`.
freq : str
An Offset String, interpreted as the frequency of
@@ -233,9 +231,9 @@ def interpolateIndex(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values and shape may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values and shape may have changed relatively to the flags input.
"""
if data[field].empty:
return data, flags
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index 844643f61856dbd5ea6027a4fba44d26a91697fb..b16e93d995311a556ccc07ffc279ab75f179f339 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -42,8 +42,8 @@ def flagByStray(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
partition_freq : str, int, or None, default None
Determines the segmentation of the data into partitions, the kNN algorithm is
@@ -146,8 +146,8 @@ def _evalStrayLabels(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the labels to be evaluated.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
fields : list[str]
A list of strings, holding the column names of the variables, the stray labels shall be
projected onto.
@@ -383,8 +383,8 @@ def flagMVScores(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged. (Here a dummy, for structural reasons)
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
fields : List[str]
List of fieldnames, corresponding to the variables that are to be included into the flagging process.
trafo : callable, default lambda x:x
@@ -437,9 +437,9 @@ def flagMVScores(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed, relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed, relatively to the flags input.
Notes
-----
@@ -540,8 +540,8 @@ def flagRaise(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
thresh : float
The threshold, for the total rise (thresh > 0), or total drop (thresh < 0), value courses must
not exceed within a timespan of length `raise_window`.
@@ -568,9 +568,9 @@ def flagRaise(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed, relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed, relatively to the flags input.
Notes
-----
@@ -700,8 +700,8 @@ def flagMAD(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged. (Here a dummy, for structural reasons)
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
window : str
Offset string. Denoting the windows size that the "Z-scored" values have to lie in.
z: float, default 3.5
@@ -713,9 +713,9 @@ def flagMAD(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed, relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed, relatively to the flags input.
References
----------
@@ -780,8 +780,8 @@ def flagOffset(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged. (Here a dummy, for structural reasons)
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
thresh : float
Minimum difference between to values, to consider the latter one as a spike. See condition (1)
tolerance : float
@@ -802,9 +802,9 @@ def flagOffset(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed, relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed, relatively to the flags input.
References
----------
@@ -933,8 +933,8 @@ def flagByGrubbs(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
winsz : {int, str}
The size of the window you want to use for outlier testing. If an integer is passed, the size
refers to the number of periods of every testing window. If a string is passed, it has to be an offset string,
@@ -955,9 +955,9 @@ def flagByGrubbs(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the flags input.
References
----------
@@ -1029,8 +1029,8 @@ def flagRange(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-flagged.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
min : float
Lower bound for valid data.
max : float
@@ -1042,9 +1042,8 @@ def flagRange(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
"""
# using .values is much faster
@@ -1085,8 +1084,8 @@ def flagCrossStatistic(
A dictionary of pandas.Series, holding all the data.
field : str
A dummy parameter.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional informations related to `data`.
+ flags : saqc.Flags
+ A flags object, holding flags and additional informations related to `data`.
fields : str
List of fieldnames in data, determining wich variables are to be included into the flagging process.
thresh : float
@@ -1105,9 +1104,9 @@ def flagCrossStatistic(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the input flagger.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the input flags.
References
----------
diff --git a/saqc/funcs/pattern.py b/saqc/funcs/pattern.py
index 5f4829e9cd612fe472b0754d3652f71a72e0e7ae..564247488131ed032efc9ebf586ccdad8f456aa8 100644
--- a/saqc/funcs/pattern.py
+++ b/saqc/funcs/pattern.py
@@ -39,8 +39,8 @@ def flagPatternByDTW(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to correct.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
ref_field: str
The fieldname in `data' which holds the pattern.
widths: tuple of int
@@ -57,9 +57,9 @@ def flagPatternByDTW(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the flags input.
References
@@ -126,8 +126,8 @@ def flagPatternByWavelet(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to correct.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
ref_field: str
The fieldname in `data` which holds the pattern.
max_distance: float
@@ -142,9 +142,9 @@ def flagPatternByWavelet(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the flags input.
References
diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py
index 967966c9c05476a0059be12b690abcbda6aee129..8bb871cddc7deb6cf8b6a762593089754647bb7d 100644
--- a/saqc/funcs/resampling.py
+++ b/saqc/funcs/resampling.py
@@ -75,8 +75,8 @@ def aggregate(
field : str
The fieldname of the column, holding the data-to-be-regularized.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.freq
+ flags : saqc.Flags
+ Container to store quality flags to data. freq
freq : str
The sampling frequency the data is to be aggregated (resampled) at.
@@ -101,9 +101,9 @@ def aggregate(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values and shape may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values and shape may have changed relatively to the flags input.
"""
data, flags = copy(data, field, flags, field + '_original')
@@ -149,8 +149,8 @@ def linear(
field : str
The fieldname of the column, holding the data-to-be-regularized.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.freq
+ flags : saqc.Flags
+ Container to store flags of the data. freq
freq : str
An offset string. The frequency of the grid you want to interpolate your data at.
@@ -160,9 +160,9 @@ def linear(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values and shape may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values and shape may have changed relatively to the flags input.
"""
data, flags = copy(data, field, flags, field + '_original')
@@ -208,8 +208,8 @@ def interpolate(
field : str
The fieldname of the column, holding the data-to-be-regularized.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.freq
+ flags : saqc.Flags
+ Container to store flags of the data. freq
freq : str
An offset string. The frequency of the grid you want to interpolate your data at.
@@ -227,9 +227,9 @@ def interpolate(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values and shape may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values and shape may have changed relatively to the flags input.
"""
data, flags = copy(data, field, flags, field + '_original')
@@ -294,8 +294,8 @@ def mapToOriginal(
field : str
The fieldname of the column, holding the data-to-be-deharmonized.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.freq
+ flags : saqc.Flags
+ Container to store flags of the data. freq
method : {'inverse_fagg', 'inverse_bagg', 'inverse_nagg', 'inverse_fshift', 'inverse_bshift', 'inverse_nshift',
'inverse_interpolation'}
@@ -307,9 +307,9 @@ def mapToOriginal(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values and shape may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values and shape may have changed relatively to the flags input.
"""
newfield = str(field) + '_original'
data, flags = reindexFlags(data, newfield, flags, method, source=field, to_mask=False)
@@ -338,8 +338,8 @@ def shift(
field : str
The fieldname of the column, holding the data-to-be-shifted.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
freq : str
An frequency Offset String that will be interpreted as the sampling rate you want the data to be shifted to.
@@ -366,9 +366,9 @@ def shift(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values and shape may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values and shape may have changed relatively to the flags input.
"""
data, flags = copy(data, field, flags, field + '_original')
return _shift(data, field, flags, freq, method=method, freq_check=freq_check, **kwargs)
@@ -459,8 +459,8 @@ def resample(
field : str
The fieldname of the column, holding the data-to-be-resampled.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
freq : str
An Offset String, that will be interpreted as the frequency you want to resample your data with.
@@ -509,9 +509,9 @@ def resample(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values and shape may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values and shape may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values and shape may have changed relatively to the flags input.
"""
flagged = _isflagged(flags[field], kwargs['to_mask'])
datcol = data[field]
@@ -651,8 +651,8 @@ def reindexFlags(
field : str
The fieldname of the data column, you want to project the source-flags onto.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
method : {'inverse_fagg', 'inverse_bagg', 'inverse_nagg', 'inverse_fshift', 'inverse_bshift', 'inverse_nshift'}
The method used for projection of source flags onto field flags. See description above for more details.
@@ -668,9 +668,9 @@ def reindexFlags(
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values and shape may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values and shape may have changed relatively to the flags input.
"""
flagscol = flags[source]
diff --git a/saqc/funcs/residues.py b/saqc/funcs/residues.py
index ad7b88a6414c5bb6876ab2c8dfcab2f09fe1c8c1..28a62acd1f3fcd13b121351e5527385a4c2af707 100644
--- a/saqc/funcs/residues.py
+++ b/saqc/funcs/residues.py
@@ -65,8 +65,8 @@ def calculatePolynomialResidues(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-modelled.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
winsz : {str, int}
The size of the window you want to use for fitting. If an integer is passed, the size
refers to the number of periods for every fitting window. If an offset string is passed,
@@ -95,9 +95,9 @@ def calculatePolynomialResidues(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the flags input.
"""
return fitPolynomial(
diff --git a/saqc/funcs/rolling.py b/saqc/funcs/rolling.py
index 4b8a5f64e2e509a17357aa2e257a3162d41e8571..db9b026fab2c84a46c522a6b79c659acfef0a4ea 100644
--- a/saqc/funcs/rolling.py
+++ b/saqc/funcs/rolling.py
@@ -37,8 +37,8 @@ def roll(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-modelled.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
winsz : {int, str}
The size of the window you want to roll with. If an integer is passed, the size
refers to the number of periods for every fitting window. If an offset string is passed,
@@ -66,9 +66,8 @@ def roll(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
"""
data = data.copy()
to_fit = data[field]
diff --git a/saqc/funcs/scores.py b/saqc/funcs/scores.py
index d5b192aa6c3a9b5b2250682e948609997663bef8..02812f44a1e821d9310bdb63abaf0ba8ab1af044 100644
--- a/saqc/funcs/scores.py
+++ b/saqc/funcs/scores.py
@@ -57,8 +57,8 @@ def assignKNNScore(
A dictionary of pandas.Series, holding all the data.
field : str
The reference variable, the deviation from wich determines the flagging.
- flagger : saqc.flagger
- A flagger object, holding flags and additional informations related to `data`.fields
+ flags : saqc.flags
+ A flags object, holding flags and additional informations related to `data`.fields
n_neighbors : int, default 10
The number of nearest neighbors to which the distance is comprised in every datapoints scoring calculation.
trafo : Callable[np.array, np.array], default lambda x: x
diff --git a/saqc/funcs/tools.py b/saqc/funcs/tools.py
index 90db8705572a16ef4fabb1d5c579b0a1405443df..f8950debeee0dcdc0f177647494f830f5329ea00 100644
--- a/saqc/funcs/tools.py
+++ b/saqc/funcs/tools.py
@@ -23,8 +23,8 @@ def copy(data: DictOfSeries, field: str, flags: Flags, new_field: str, **kwargs)
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to fork (copy).
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
new_field: str
Target name.
@@ -32,10 +32,10 @@ def copy(data: DictOfSeries, field: str, flags: Flags, new_field: str, **kwargs)
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- data shape may have changed relatively to the flagger input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags shape may have changed relatively to the flagger input.
+ data shape may have changed relatively to the flags input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags shape may have changed relatively to the flags input.
"""
if new_field in flags.columns.union(data.columns):
raise ValueError(f"{field}: field already exist")
@@ -49,7 +49,7 @@ def copy(data: DictOfSeries, field: str, flags: Flags, new_field: str, **kwargs)
@register(masking='none', module="tools")
def drop(data: DictOfSeries, field: str, flags: Flags, **kwargs) -> Tuple[DictOfSeries, Flags]:
"""
- The function drops field from the data dios and the flagger.
+ The function drops field from the data dios and the flags.
Parameters
----------
@@ -57,17 +57,17 @@ def drop(data: DictOfSeries, field: str, flags: Flags, **kwargs) -> Tuple[DictOf
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to drop.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
Returns
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- data shape may have changed relatively to the flagger input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags shape may have changed relatively to the flagger input.
+ data shape may have changed relatively to the flags input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags shape may have changed relatively to the flags input.
"""
del data[field]
del flags[field]
@@ -77,7 +77,7 @@ def drop(data: DictOfSeries, field: str, flags: Flags, **kwargs) -> Tuple[DictOf
@register(masking='none', module="tools")
def rename(data: DictOfSeries, field: str, flags: Flags, new_name: str, **kwargs) -> Tuple[DictOfSeries, Flags]:
"""
- The function renames field to new name (in both, the flagger and the data).
+ The function renames field to new name (in both, the flags and the data).
Parameters
----------
@@ -85,8 +85,8 @@ def rename(data: DictOfSeries, field: str, flags: Flags, new_name: str, **kwargs
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the data column, you want to rename.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
new_name : str
String, field is to be replaced with.
@@ -94,8 +94,8 @@ def rename(data: DictOfSeries, field: str, flags: Flags, new_name: str, **kwargs
-------
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ The quality flags of data
"""
data[new_name] = data[field]
flags.history[new_name] = flags.history[field]
@@ -141,8 +141,8 @@ def mask(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-masked.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store flags of the data.
mode : {"periodic", "mask_var"}
The masking mode.
- "periodic": parameters "period_start", "period_end" are evaluated to generate a periodical mask
@@ -170,9 +170,9 @@ def mask(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
- Flags values may have changed relatively to the flagger input.
+ flags : saqc.Flags
+ The quality flags of data
+ Flags values may have changed relatively to the flags input.
Examples
diff --git a/saqc/funcs/transformation.py b/saqc/funcs/transformation.py
index 48a072909dc3235c46e0baa21f29217f9afda5ca..91952d0f115aceb763ce6e5851e9938adf0ec5d5 100644
--- a/saqc/funcs/transformation.py
+++ b/saqc/funcs/transformation.py
@@ -29,8 +29,8 @@ def transform(
A dictionary of pandas.Series, holding all the data.
field : str
The fieldname of the column, holding the data-to-be-transformed.
- flagger : saqc.flagger.Flagger
- A flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ Container to store quality flags to data.
func : Callable[{pd.Series, np.array}, np.array]
Function to transform data[field] with.
partition_freq : {None, float, str}, default None
@@ -46,8 +46,8 @@ def transform(
data : dios.DictOfSeries
A dictionary of pandas.Series, holding all the data.
Data values may have changed relatively to the data input.
- flagger : saqc.flagger.Flagger
- The flagger object, holding flags and additional Informations related to `data`.
+ flags : saqc.Flags
+ The quality flags of data
"""
data = data.copy()
diff --git a/sphinx-doc/make_doc_module.py b/sphinx-doc/make_doc_module.py
index b2f8ff48462dca6456abb4f993f8874ac2860628..1e508a2539402ebd22d102223bf75214daf4b71c 100644
--- a/sphinx-doc/make_doc_module.py
+++ b/sphinx-doc/make_doc_module.py
@@ -83,7 +83,7 @@ def parse_func_dcstrings(m_paths):
continue
dcstr = rm_section(dcstr, 'Returns')
dcstr = rm_parameter(dcstr, 'data')
- dcstr = rm_parameter(dcstr, 'flagger')
+ dcstr = rm_parameter(dcstr, 'flags')
parameters = get_parameter(dcstr)
parameters = [f"{p[0]}={p[1]}" if p[1] else p[0] for p in parameters]
signature = f"def {func.name}({', '.join(parameters)}):"
diff --git a/tests/common.py b/tests/common.py
index 1a3f501a768762130c05bd8e1f75830982255fad..21fc6c9c211bb3450ebc48e58fff162a0a771d95 100644
--- a/tests/common.py
+++ b/tests/common.py
@@ -11,7 +11,6 @@ from saqc.core import initFlagsLike, Flags
TESTNODATA = (np.nan, -9999)
-TESTFLAGGER = (Flags(),)
def flagAll(data, field, flags, **kwargs):
@@ -47,30 +46,30 @@ def checkDataFlagsInvariants(data, flags, field, identical=True):
Check all invariants that must hold at any point for
* field
* data
- * flagger
+ * flags
* data[field]
- * flagger[field]
+ * flags[field]
* data[field].index
- * flagger[field].index
- * between data and flagger
- * between data[field] and flagger[field]
+ * flags[field].index
+ * between data and flags
+ * between data[field] and flags[field]
Parameters
----------
data : dios.DictOfSeries
data container
- flagger : Flags
+ flags : Flags
flags container
field : str
the field in question
identical : bool, default True
- whether to check indexes of data and flagger to be
+ whether to check indexes of data and flags to be
identical (True, default) of just for equality.
"""
assert isinstance(data, dios.DictOfSeries)
assert isinstance(flags, Flags)
- # all columns in data are in flagger
+ # all columns in data are in flags
assert data.columns.difference(flags.columns).empty
# ------------------------------------------------------------------------
diff --git a/tests/core/test_flagger.py b/tests/core/test_flagger.py
deleted file mode 100644
index 1af9f47106458bcdfe1c2016d43c15f22a630901..0000000000000000000000000000000000000000
--- a/tests/core/test_flagger.py
+++ /dev/null
@@ -1,756 +0,0 @@
-#!/usr/bin/env python
-
-import pytest
-import numpy as np
-import pandas as pd
-from pandas.api.types import is_bool_dtype
-
-import dios
-
-from tests.common import TESTFLAGGER, initData
-
-
-pytestmark = pytest.mark.skip('old flagger tests - rewrite needed')
-
-
-def _getDataset(rows, cols):
- return initData(cols=cols, rows=rows, start_date="2011-01-01", end_date="2011-01-10")
-
-
-DATASETS = [
- _getDataset(0, 1),
- _getDataset(1, 1),
- _getDataset(100, 1),
- # _getDataset(1000, 1),
- _getDataset(0, 4),
- _getDataset(1, 4),
- # _getDataset(100, 4),
- # _getDataset(1000, 4),
- # _getDataset(10000, 40),
- _getDataset(20, 4),
-]
-
-
-def check_all_dios_index_length(tocheck, expected):
- for c in tocheck:
- if len(tocheck[c]) != len(expected[c]):
- return False
- return True
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_initFlags(data, flagger):
- """
- test before:
- - None
- """
-
- newflagger = flagger.initFlags(data)
- assert isinstance(newflagger, type(flagger))
- assert newflagger is not flagger
-
- flags = newflagger.getFlags()
- assert isinstance(flags, dios.DictOfSeries)
-
- assert len(flags.columns) >= len(data.columns)
- assert check_all_dios_index_length(flags, data)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_initFlagsWithFlags(data, flagger):
- flags = dios.DictOfSeries(pd.Series(data=flagger.BAD))
- flagger = flagger.initFlags(flags=flags)
- assert (flagger.flags == flags).all(axis=None)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_getFlags(data, flagger):
- """
- test before:
- - initFlags()
-
- we need to check:
- - access all flags -> get a dios
- - access some columns of flags -> get a dios
- - access one column of flags -> get a series
- """
-
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- # all - dios
- flags0 = flagger.getFlags()
- assert isinstance(flags0, dios.DictOfSeries)
- assert (flags0.columns == data.columns).all()
- assert check_all_dios_index_length(flags0, data)
- for dt in flags0.dtypes:
- assert dt == flagger.dtype
-
- # some - dios
- if len(data.columns) >= 2:
- cols = data.columns[:2].to_list()
- flags1 = flagger.getFlags(cols)
- assert isinstance(flags1, dios.DictOfSeries)
- assert (flags1.columns == data.columns[:2]).all()
- assert check_all_dios_index_length(flags1, data[cols])
- for dt in flags1.dtypes:
- assert dt == flagger.dtype
-
- # series
- flags2 = flagger.getFlags(field)
- assert isinstance(flags2, pd.Series)
- assert flags2.dtype == flagger.dtype
- assert flags2.shape[0] == data[field].shape[0]
- # NOTE: need fix in dios see issue #16 (has very low priority)
- # assert flags2.name in data.columns
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_setFlags(data, flagger):
- """
- test before:
- - initFlags()
- - getFlags()
- """
- flagger = flagger.initFlags(data)
- sl = slice("2011-01-02", "2011-01-05")
- field, *_ = data.columns
-
- base = flagger.getFlags()
-
- flagger_good = flagger.setFlags(field, flag=flagger.GOOD, loc=sl)
- assert isinstance(flagger_good, type(flagger))
- assert flagger_good is not flagger
-
- flags_good = flagger_good.getFlags()
- assert len(flags_good[field]) <= len(base[field])
- assert (flags_good.columns == base.columns).all()
- assert (flags_good.loc[sl, field] == flagger.GOOD).all()
-
- # overflag works BAD > GOOD
- flagger_bad = flagger_good.setFlags(field, flag=flagger.BAD)
- assert (flagger_bad.getFlags(field) == flagger.BAD).all()
-
- # overflag doesn't work GOOD < BAD
- flagger_still_bad = flagger_bad.setFlags(field, flag=flagger.GOOD)
- assert (flagger_still_bad.getFlags(field) == flagger.BAD).all()
-
- # overflag does work with force
- flagger_forced_good = flagger_bad.setFlags(field, flag=flagger.GOOD, force=True)
- assert (flagger_forced_good.getFlags(field) == flagger.GOOD).all()
-
- with pytest.raises(ValueError):
- flagger.setFlags(field=None, flag=flagger.BAD)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_sliceFlagger(data, flagger):
- """
- test before:
- - initFlags()
- - getFlags() inside slice()
- """
- sl = slice(None, None, 3)
-
- flagger = flagger.initFlags(data)
- newflagger = flagger.slice(loc=sl)
- assert isinstance(newflagger, type(flagger))
-
- newflags = newflagger.getFlags()
- assert (newflags.columns == data.columns).all()
- assert check_all_dios_index_length(newflags, data[sl])
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_sliceFlaggerDrop(data, flagger):
- flagger = flagger.initFlags(data)
- with pytest.raises(TypeError):
- flagger.getFlags(field=data.columns, drop="var")
-
- field = data.columns[0]
- expected = data.columns.drop(field)
-
- filtered = flagger.slice(drop=field)
- assert (filtered.getFlags().columns == expected).all(axis=None)
- assert (filtered.getFlags().to_df().index == data[expected].to_df().index).all(axis=None)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_mergeFlagger(data, flagger):
- """
- test before:
- - initFlags()
- - getFlags()
- - setFlags()
- - slice()
- """
- field, *_ = data.columns
- sl = slice(None, None, 3)
-
- this_flagger = flagger.initFlags(data)
- other_flagger = this_flagger.slice(loc=sl).setFlags(field)
- result_flagger = this_flagger.merge(other_flagger)
-
- result_flags = result_flagger.getFlags()
- other_flags = other_flagger.getFlags()
-
- # check flags that was set
- check = result_flags.loc[sl, field] == other_flags[field]
- assert check.all(None)
- # check flags that was not set
- mask = ~result_flags[field].index.isin(other_flags[field].index)
- check = result_flags.loc[mask, field] == result_flagger.UNFLAGGED
- assert check.all(None)
-
- # check unchanged columns
- cols = data.columns.to_list()
- cols.remove(field)
- check = result_flags[cols] == result_flagger.UNFLAGGED
- assert check.all(None)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_mergeFlaggerColumnsDiff(data, flagger):
- """
- test before:
- - initFlags()
- - getFlags()
- - setFlags()
- - slice()
- - merge()
- """
- field, *_ = data.columns
- new_field = field + "_new"
- sl = slice(None, None, 2)
-
- other_data = data.loc[sl]
- other_data.columns = [new_field] + data.columns[1:].to_list()
- other_flagger = flagger.initFlags(other_data)
-
- this_flagger = flagger.initFlags(data).setFlags(field, flag=flagger.BAD)
- result_flagger = this_flagger.merge(other_flagger)
-
- result_flags = result_flagger.getFlags()
- other_flags = other_flagger.getFlags()
-
- # we need to check if
- # - the new column is present
- # - the new column is identical to the original
- # - the other column are unchanged
- # - field-column is BAD
- # - other columns are UNFLAGGED
-
- assert new_field in result_flags
-
- check = result_flags[new_field] == other_flags[new_field]
- assert check.all(None)
-
- check = result_flags[field] == result_flagger.BAD
- assert check.all(None)
-
- cols = data.columns.to_list()
- cols.remove(field)
- check = result_flags[cols] == result_flagger.UNFLAGGED
- assert check.all(None)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_mergeFlaggerIndexDiff(data, flagger):
- """
- test before:
- - initFlags()
- - getFlags()
- - setFlags()
- - slice()
- - merge()
-
- we need to check:
- - index is union of this and other's index
- - indices + values that only in this, should be present
- - indices + values that only in other, should be present
- - indices that in this and other, have values from other
- """
- field, *_ = data.columns
- sl = slice(None, None, 2)
-
- def shiftindex(s):
- s.index = s.index + pd.Timedelta(minutes=2, seconds=25)
- return s
-
- # create a sliced time-shifted version of data
- other_data = data.loc[sl].apply(shiftindex)
- if isinstance(other_data, pd.Series):
- pass
-
- this_flagger = flagger.initFlags(data).setFlags(field, flag=flagger.BAD)
- other_flagger = flagger.initFlags(other_data)
- result_flagger = this_flagger.merge(other_flagger)
-
- result_flags = result_flagger.getFlags()
- this_flags = this_flagger.getFlags()
- other_flags = other_flagger.getFlags()
-
- for c in result_flags:
- t, o, r = this_flags[c], other_flags[c], result_flags[c]
- assert (r.index == t.index.union(o.index)).all()
-
- only_this = t.index.difference(o.index)
- only_other = o.index.difference(t.index)
- both = t.index.intersection(o.index)
-
- # nothing is missing
- assert (r.index == only_this.union(only_other).union(both)).all()
-
- assert (r[only_this] == t[only_this]).all()
- assert (r[only_other] == o[only_other]).all()
- assert (r[both] == o[both]).all()
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_mergeFlaggerOuter(data, flagger):
-
- field = data.columns[0]
-
- data_left = data
- data_right = data.iloc[::2]
-
- left = flagger.initFlags(data=data_left).setFlags(field=field, flag=flagger.BAD)
-
- right = flagger.initFlags(data=data_right).setFlags(field, flag=flagger.GOOD)
-
- merged = left.merge(right, join="outer")
-
- loc = data_right[field].index.difference(data_left[field].index)
- assert (merged.getFlags(field, loc=loc) == flagger.GOOD).all(axis=None)
- assert (merged.getFlags(field, loc=data_left[field].index) == flagger.BAD).all(axis=None)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_mergeFlaggerInner(data, flagger):
-
- field = data.columns[0]
-
- data_left = data
- data_right = data.iloc[::2]
-
- left = flagger.initFlags(data=data_left).setFlags(field=field, flag=flagger.BAD)
-
- right = flagger.initFlags(data=data_right).setFlags(field, flag=flagger.GOOD)
-
- merged = left.merge(right, join="inner")
-
- assert (merged.getFlags(field).index == data_right[field].index).all()
- assert (merged.getFlags(field) == flagger.BAD).all()
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_mergeFlaggerMerge(data, flagger):
-
- field = data.columns[0]
- data_left = data
- data_right = data.iloc[::2]
-
- left = flagger.initFlags(data=data_left).setFlags(field=field, flag=flagger.BAD)
-
- right = flagger.initFlags(data=data_right).setFlags(field, flag=flagger.GOOD)
-
- merged = left.merge(right, join="merge")
-
- loc = data_left[field].index.difference(data_right[field].index)
- assert (merged.getFlags(field, loc=data_right[field].index) == flagger.GOOD).all(axis=None)
- assert (merged.getFlags(field, loc=loc) == flagger.BAD).all(axis=None)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_isFlaggedDios(data, flagger):
- """
- test before:
- - initFlags()
- - setFlags()
- """
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- mask = np.zeros(len(data[field]), dtype=bool)
-
- df_tests = [
- (flagger.isFlagged(), mask),
- (flagger.setFlags(field).isFlagged(), ~mask),
- (flagger.setFlags(field, flag=flagger.GOOD).isFlagged(flag=flagger.GOOD, comparator=">"), mask,),
- (flagger.setFlags(field, flag=flagger.GOOD).isFlagged(flag=flagger.GOOD, comparator="<"), mask,),
- (flagger.setFlags(field, flag=flagger.GOOD).isFlagged(flag=flagger.GOOD, comparator="=="), ~mask,),
- ]
- for flags, expected in df_tests:
- assert np.all(flags[field] == expected)
- assert isinstance(flags, dios.DictOfSeries)
- assert check_all_dios_index_length(flags, data)
- assert (flags.columns == data.columns).all()
- for dt in flags.dtypes:
- assert is_bool_dtype(dt)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_isFlaggedSeries(data, flagger):
- """
- test before:
- - initFlags()
- - setFlags()
- """
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- mask = np.zeros(len(data[field]), dtype=bool)
-
- series_tests = [
- (flagger.isFlagged(field), mask),
- (flagger.setFlags(field).isFlagged(field), ~mask),
- (flagger.setFlags(field, flag=flagger.GOOD).isFlagged(field, flag=flagger.GOOD, comparator=">"), mask,),
- (flagger.setFlags(field, flag=flagger.GOOD).isFlagged(field, flag=flagger.GOOD, comparator="<"), mask,),
- (flagger.setFlags(field, flag=flagger.GOOD).isFlagged(field, flag=flagger.GOOD, comparator="=="), ~mask,),
- ]
- for flags, expected in series_tests:
- assert np.all(flags == expected)
- assert isinstance(flags, pd.Series)
- assert flags.dtype == bool
- assert flags.shape[0] == data[field].shape[0]
- # NOTE: need fix in dios see issue #16 (has very low priority)
- # assert flags.name in data.columns
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_isFlaggedSeries_fail(data, flagger):
- """
- test before:
- - initFlags()
- """
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- fail_tests = [
- {"flag": pd.Series(index=data[field].index, data=flagger.BAD).astype(flagger.dtype)},
- # NOTE: allowed since use of dios
- # {"field": ["var1", "var2"]},
- ]
- for args in fail_tests:
- with pytest.raises(TypeError):
- flagger.isFlagged(**args)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_clearFlags(data, flagger):
- """
- test before:
- - initFlags()
- - getFlags()
- - setFlags()
- - isFlagged()
- """
- flagger = flagger.initFlags(data)
- sl = slice("2011-01-02", "2011-01-05")
- field, *_ = data.columns
-
- base = flagger.getFlags(field)
-
- flagger = flagger.setFlags(field=field, flag=flagger.BAD)
- assert np.sum(flagger.isFlagged(field)) == len(base)
-
- flaggernew = flagger.clearFlags(field)
- assert isinstance(flaggernew, type(flagger))
- assert flaggernew is not flagger
- assert len(flagger.getFlags(field)) == len(data[field])
-
- flagger = flagger.clearFlags(field)
- assert np.sum(flagger.isFlagged(field)) == 0
- assert len(flagger.getFlags(field)) == len(data[field])
-
- flagger = flagger.setFlags(field=field, flag=flagger.BAD)
- assert np.sum(flagger.isFlagged(field)) == len(base)
- assert len(flagger.getFlags(field)) == len(data[field])
-
- flagger = flagger.clearFlags(field, loc=sl)
- assert len(flagger.getFlags(field)) == len(data[field])
- unflagged = flagger.isFlagged(field, loc=sl)
- assert np.sum(unflagged) == 0
- assert np.sum(flagger.isFlagged(field)) == len(data[field]) - len(unflagged)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_dtype(data, flagger):
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- tests = (
- flagger.getFlags(field).astype(str),
- "TEST",
- 55,
- )
-
- for test in tests:
- with pytest.raises(TypeError):
- flagger = flagger.setFlags(field, flag=test)
- assert flagger.getFlags(field).dtype == flagger.dtype
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER[-1:])
-def test_returnCopy(data, flagger):
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- base = flagger.getFlags()
-
- assert flagger.getFlags() is not base
- assert flagger.isFlagged() is not base
- assert flagger.setFlags(field) is not flagger
- assert flagger.clearFlags(field) is not flagger
-
-
-LOC_ILOC_FUNCS = ["isFlagged", "getFlags"]
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-@pytest.mark.parametrize("flaggerfunc", LOC_ILOC_FUNCS)
-def test_loc(data, flagger, flaggerfunc):
- flagger = flagger.initFlags(data)
- sl = slice("2011-01-02", "2011-01-05")
- field, *_ = data.columns
-
- chunk = data.loc[sl, field]
- d = data.loc[sl]
- if d.empty:
- return
-
- m = data[field].index.get_loc(d[field].index[0])
- M = data[field].index.get_loc(d[field].index[-1])
- mask = np.full(len(data[field]), False)
- mask[m:M] = True
-
- flagger_func = getattr(flagger, flaggerfunc)
-
- # masked
- mflags0 = flagger_func(field, loc=mask)
- mflags1 = flagger_func().loc[mask, field]
- mflags2 = flagger_func(field).loc[mask]
- mflags3 = flagger_func(loc=mask)[field]
-
- assert (mflags0 == mflags1).all()
- assert (mflags0 == mflags2).all()
- assert (mflags0 == mflags3).all()
-
- # indexed
- iflags0 = flagger_func(field, loc=chunk.index)
- iflags1 = flagger_func().loc[chunk.index, field]
- iflags2 = flagger_func(field).loc[chunk.index]
- iflags3 = flagger_func(loc=chunk.index)[field]
- assert (iflags0 == iflags1).all()
- assert (iflags0 == iflags2).all()
- assert (iflags0 == iflags3).all()
-
- # sliced
- sflags0 = flagger_func(field, loc=sl)
- sflags1 = flagger_func().loc[sl, field]
- sflags2 = flagger_func(field).loc[sl]
- sflags3 = flagger_func(loc=sl)[field]
- assert (sflags0 == sflags1).all()
- assert (sflags0 == sflags2).all()
- assert (sflags0 == sflags3).all()
-
- assert (sflags0 == iflags0).all()
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_classicUseCases(data, flagger):
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- flagger = flagger.clearFlags(field)
-
- # data-mask, same length than flags
- d = data[field]
- mask = d < (d.max() - d.min()) // 2
- flagged = flagger.setFlags(field, loc=mask, flag=flagger.BAD).isFlagged(field)
- assert (flagged == mask).all()
-
- flagger = flagger.clearFlags(field)
-
- indices = np.arange(0, len(data[field]))
- mask = indices % 3 == 0
- indices = indices[mask]
- # we had some fun with numpy and end up with
- # numpy indices (positional), but with different length..
- # make dt-index with iloc, then pass to loc
- dt_idx = data[field].iloc[indices].index
- flagged = flagger.setFlags(field, loc=dt_idx, flag=flagger.BAD).isFlagged(field)
- assert (flagged.iloc[indices] == flagged[flagged]).all()
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_getFlagsWithExtras(data, flagger):
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- flags, extra = flagger.getFlags(field, full=True)
- assert isinstance(flags, pd.Series)
- assert isinstance(extra, dict)
- for k, v in extra.items():
- assert isinstance(v, pd.Series)
- assert flags.index.equals(v.index)
-
- flags, extra = flagger.getFlags(full=True)
- assert isinstance(flags, dios.DictOfSeries)
- assert isinstance(extra, dict)
- for k, v in extra.items():
- assert isinstance(v, dios.DictOfSeries)
- assert flags.columns.equals(v.columns)
- for c in flags:
- assert flags[c].index.equals(v[c].index)
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_replace_delete(data, flagger):
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
- newflagger = flagger.replaceField(field=field, flags=None)
-
- new, newextra = newflagger.getFlags(full=True)
- assert field not in newflagger.flags
- for k in newextra:
- assert field not in newextra[k]
-
- with pytest.raises(ValueError):
- flagger.replaceField(field="i_dont_exist", flags=None)
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_replace_insert(data, flagger):
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
- newfield = 'fooo'
- flags, extra = flagger.getFlags(field, full=True)
- newflagger = flagger.replaceField(field=newfield, flags=flags, **extra)
- old, oldextra = flagger.getFlags(full=True)
- new, newextra = newflagger.getFlags(full=True)
- assert newfield in newflagger.flags
- assert (newflagger._flags[newfield] == flagger._flags[field]).all()
- assert newflagger._flags[newfield] is not flagger._flags[field] # not a copy
- for k in newextra:
- assert newfield in newextra[k]
- assert (newextra[k][newfield] == oldextra[k][field]).all()
-
-
-@pytest.mark.parametrize("data", DATASETS)
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_replace_replace(data, flagger):
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
- flags, extra = flagger.getFlags(field, full=True)
-
- # set everything to DOUBTFUL
- flags[:] = flagger.BAD
- for k, v in extra.items():
- v[:] = flagger.BAD
- extra[k] = v
-
- newflagger = flagger.replaceField(field=field, flags=flags, **extra)
-
- old, oldextra = flagger.getFlags(full=True)
- new, newextra = newflagger.getFlags(full=True)
- assert old.columns.equals(new.columns)
- assert (new[field] == flagger.BAD).all()
-
- assert oldextra.keys() == newextra.keys()
- for k in newextra:
- o, n = oldextra[k], newextra[k]
- assert n.columns.equals(o.columns)
- assert (n[field] == flagger.BAD).all()
-
-
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_flagAfter(flagger):
- idx = pd.date_range("2000", "2001", freq='1M')
- s = pd.Series(0, index=idx)
- data = dios.DictOfSeries(s, columns=['a'])
- exp_base = pd.Series(flagger.UNFLAGGED, index=idx)
-
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- flags = flagger.setFlags(field, loc=s.index[3], flag_after=5).getFlags(field)
- exp = exp_base.copy()
- exp.iloc[3: 3+5+1] = flagger.BAD
- assert (flags == exp).all()
-
- flags = flagger.setFlags(field, loc=s.index[3], flag_after=5, win_flag=flagger.GOOD).getFlags(field)
- exp = exp_base.copy()
- exp.iloc[3: 3+5+1] = flagger.GOOD
- exp[3] = flagger.BAD
- assert (flags == exp).all()
-
- # 3 month < 99 days < 4 month
- flags = flagger.setFlags(field, loc=s.index[3], flag_after="99d").getFlags(field)
- exp = exp_base.copy()
- exp.iloc[3: 3+3+1] = flagger.BAD
- assert (flags == exp).all()
-
- # 3 month < 99 days < 4 month
- flags = flagger.setFlags(field, loc=s.index[3], flag_after="99d", win_flag=flagger.GOOD).getFlags(field)
- exp = exp_base.copy()
- exp.iloc[3: 3+3+1] = flagger.GOOD
- exp[3] = flagger.BAD
- assert (flags == exp).all()
-
-
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_flagBefore(flagger):
- idx = pd.date_range("2000", "2001", freq='1M')
- s = pd.Series(0, index=idx)
- data = dios.DictOfSeries(s, columns=['a'])
- exp_base = pd.Series(flagger.UNFLAGGED, index=idx)
-
- flagger = flagger.initFlags(data)
- field, *_ = data.columns
-
- flags = flagger.setFlags(field, loc=s.index[8], flag_before=5).getFlags(field)
- exp = exp_base.copy()
- exp.iloc[8-5: 8+1] = flagger.BAD
- assert (flags == exp).all()
-
- flags = flagger.setFlags(field, loc=s.index[8], flag_before=5, win_flag=flagger.GOOD).getFlags(field)
- exp = exp_base.copy()
- exp.iloc[8-5: 8+1] = flagger.GOOD
- exp[8] = flagger.BAD
- assert (flags == exp).all()
-
- # 3 month < 99 days < 4 month
- flags = flagger.setFlags(field, loc=s.index[8], flag_before="99d").getFlags(field)
- exp = exp_base.copy()
- exp.iloc[8-3: 8+1] = flagger.BAD
- assert (flags == exp).all()
-
- # 3 month < 99 days < 4 month
- flags = flagger.setFlags(field, loc=s.index[8], flag_before="99d", win_flag=flagger.GOOD).getFlags(field)
- exp = exp_base.copy()
- exp.iloc[8-3: 8+1] = flagger.GOOD
- exp[8] = flagger.BAD
- assert (flags == exp).all()
diff --git a/tests/fuzzy/test_masking.py b/tests/fuzzy/test_masking.py
index be77e9872f5222e6cc332dc31236081f2c25a28e..cf00f1d984d5431d4f8f09bc4345548e519f55df 100644
--- a/tests/fuzzy/test_masking.py
+++ b/tests/fuzzy/test_masking.py
@@ -48,9 +48,9 @@ def test_dataMutationPreventsUnmasking(data_field_flags):
@settings(max_examples=MAX_EXAMPLES, deadline=None)
@given(data_field_flags=dataFieldFlags())
def test_flagsMutationPreventsUnmasking(data_field_flags):
- """ test if (un)masking works as expected on flagger-changes.
+ """ test if (un)masking works as expected on flags-changes.
- if `flagger` is mutated after `_maskData`, `_unmaskData` should be a no-op
+ if `flags` is mutated after `_maskData`, `_unmaskData` should be a no-op
"""
data_in, field, flags = data_field_flags
data_masked, mask = _maskData(data_in, flags, columns=[field], to_mask=BAD)
@@ -73,7 +73,7 @@ def test_reshapingPreventsUnmasking(data_field_flags):
data_in, field, flags = data_field_flags
data_masked, mask = _maskData(data_in, flags, columns=[field], to_mask=BAD)
- # mutate indexes of `data` and `flagger`
+ # mutate indexes of `data` and `flags`
index = data_masked[field].index.to_series()
index.iloc[-len(data_masked[field])//2:] += pd.Timedelta("7.5Min")
data_masked[field] = pd.Series(data=filler, index=index)
@@ -98,8 +98,8 @@ def test_unmaskingInvertsMasking(data_field_flags):
# @settings(max_examples=MAX_EXAMPLES, deadline=None)
-# @given(data_field_flagger=dataFieldFlagger(), func_kwargs=flagFuncsKwargs())
-# def test_maskingPreservesData(data_field_flagger, func_kwargs):
+# @given(data_field_flags=dataFieldFlags(), func_kwargs=flagFuncsKwargs())
+# def test_maskingPreservesData(data_field_flags, func_kwargs):
# """
# no mutations on pre-flagged data
@@ -108,20 +108,20 @@ def test_unmaskingInvertsMasking(data_field_flags):
# are removed
# """
-# data_in, field, flagger = data_field_flagger
+# data_in, field, flags = data_field_flags
-# data_masked, mask = _maskData(data_in, flagger, columns=[field], to_mask=flagger.BAD)
+# data_masked, mask = _maskData(data_in, flags, columns=[field], to_mask=flags.BAD)
# func, kwargs = func_kwargs
-# data_masked, _ = func(data_masked, field, flagger, **kwargs)
-# data_out = _unmaskData(data_in, mask, data_masked, flagger, to_mask=flagger.BAD)
+# data_masked, _ = func(data_masked, field, flags, **kwargs)
+# data_out = _unmaskData(data_in, mask, data_masked, flags, to_mask=flags.BAD)
-# flags_in = flagger.isFlagged(flag=flagger.BAD)
+# flags_in = flags.isFlagged(flag=flags.BAD)
# assert data_in.aloc[flags_in].equals(data_out.aloc[flags_in])
# @settings(max_examples=MAX_EXAMPLES, deadline=None)
-# @given(data_field_flagger=dataFieldFlagger(), func_kwargs=flagFuncsKwargs())
-# def test_maskingEqualsRemoval(data_field_flagger, func_kwargs):
+# @given(data_field_flags=dataFieldFlags(), func_kwargs=flagFuncsKwargs())
+# def test_maskingEqualsRemoval(data_field_flags, func_kwargs):
# """
# calling a function on pre-flagged data should yield the same
# results as calling this function on data where the flagged values
@@ -129,17 +129,17 @@ def test_unmaskingInvertsMasking(data_field_flags):
# """
# func, kwargs = func_kwargs
-# data, field, flagger = data_field_flagger
-# flagged_in = flagger.isFlagged(flag=flagger.BAD, comparator=">=")
+# data, field, flags = data_field_flags
+# flagged_in = flags.isFlagged(flag=flags.BAD, comparator=">=")
# # mask and call
-# data_left, _ = _maskData(data, flagger, columns=[field], to_mask=flagger.BAD)
-# data_left, _ = func(data_left, field, flagger, **kwargs)
+# data_left, _ = _maskData(data, flags, columns=[field], to_mask=flags.BAD)
+# data_left, _ = func(data_left, field, flags, **kwargs)
# # remove and call
# data_right = data.aloc[~flagged_in]
-# flagger_right = flagger.initFlags(flagger.getFlags().aloc[~flagged_in])
-# data_right, _ = func(data_right, field, flagger_right, **kwargs)
+# flags_right = flags.initFlags(flags.getFlags().aloc[~flagged_in])
+# data_right, _ = func(data_right, field, flags_right, **kwargs)
# # NOTE: we need to handle the implicit type conversion in `_maskData`
# data_left_compare = data_left.aloc[~flagged_in]