diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d9abb9f9ff6a5bada6ab6b69c16586eb2c1464fd..78bc10f9eebfb2a6ea73c9214f6e1aaa81ba9e0b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -35,6 +35,8 @@ default:
- pip install --upgrade pip
- pip install -r requirements.txt
- pip install -r tests/requirements.txt
+ - apt update
+ - apt install -y xvfb
# ===========================================================
# Compliance stage
@@ -75,6 +77,8 @@ coverage:
stage: test
allow_failure: true
script:
+ - export DISPLAY=:99
+ - Xvfb :99 &
- pip install pytest-cov coverage
- pytest --cov=saqc tests --ignore=tests/fuzzy -Werror
after_script:
@@ -93,6 +97,8 @@ python39:
stage: test
image: python:3.9
script:
+ - export DISPLAY=:99
+ - Xvfb :99 &
- pytest tests -Werror --junitxml=report.xml
- python -m saqc --config docs/resources/data/config.csv --data docs/resources/data/data.csv --outfile /tmp/test.csv
artifacts:
@@ -105,6 +111,8 @@ python310:
stage: test
image: python:3.10
script:
+ - export DISPLAY=:99
+ - Xvfb :99 &
- pytest tests -Werror --junitxml=report.xml
- python -m saqc --config docs/resources/data/config.csv --data docs/resources/data/data.csv --outfile /tmp/test.csv
artifacts:
@@ -116,6 +124,8 @@ python311:
stage: test
image: python:3.11
script:
+ - export DISPLAY=:99
+ - Xvfb :99 &
- pytest tests -Werror --junitxml=report.xml
- python -m saqc --config docs/resources/data/config.csv --data docs/resources/data/data.csv --outfile /tmp/test.csv
artifacts:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bfc336e9f6f83dfab4373d3d703604f079b03f8a..0f09eccdf09c944bd13dc1bd003de1e65ecd6933 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,17 +11,23 @@ SPDX-License-Identifier: GPL-3.0-or-later
- `flagGeneric`: target broadcasting
- `SaQC`: automatic translation of incoming flags
- Option to change the flagging scheme after initialization
+- `flagByClick`: manually assign flags using a graphical user interface
- `SaQC`: support for selection, slicing and setting of items by use of subscription on SaQC objects (e.g. `qc[key]` and `qc[key] = value`).
Selection works with single keys, collections of keys and string slices (e.g. `qc["a":"f"]`). Values can be SaQC objects, pd.Series,
Iterable of Series and dict-like with series values.
-- `transferFlags` becomes a multivariate function
+- `transferFlags` is a multivariate function
+- `plot`: added `yscope` keyword
+- `setFlags`: function to replace `flagManual`
+- `flagUniLOF`: added defaultly applied correction to mitigate phenomenon of overflagging at relatively steep data value slopes. (parameter `slope_correct`).
### Changed
### Removed
### Fixed
+- `flagConstants`: fixed flagging of rolling ramps
- `Flags`: add meta entry to imported flags
- group operations were overwriting existing flags
- `SaQC._construct` : was not working for inherit classes (used hardcoded `SaQC` to construct a new instance).
### Deprecated
+- `flagManual` in favor of `setFlags`
## [2.5.0](https://git.ufz.de/rdm-software/saqc/-/tags/v2.4.1) - 2023-06-22
[List of commits](https://git.ufz.de/rdm-software/saqc/-/compare/v2.4.1...v2.5.0)
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 2aeaac87678dc41f9dc1d8b1eb919ec92a0300a3..8c277e95aa5b807e6b859d04b6c5b40dd86dd4b1 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -4,11 +4,11 @@
recommonmark==0.7.1
sphinx==7.2.6
-sphinx-automodapi==0.16.0
+sphinx-automodapi==0.17.0
sphinxcontrib-fulltoc==1.2.0
sphinx-markdown-tables==0.0.17
jupyter-sphinx==0.5.3
-sphinx_autodoc_typehints==1.25.2
-sphinx-tabs==3.4.4
+sphinx_autodoc_typehints==2.0.0
+sphinx-tabs==3.4.5
sphinx-design==0.5.0
-pydata-sphinx-theme==0.14.4
+pydata-sphinx-theme==0.15.2
diff --git a/requirements.txt b/requirements.txt
index 4d8b456b9988f3e05b7885dea4a0e5d052f261d0..c9553ab49916a6ba411ae6533e51b90a48285995 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,12 +5,12 @@
Click==8.1.7
docstring_parser==0.15
dtw==1.4.0
-matplotlib==3.8.2
-numpy==1.26.2
+matplotlib==3.8.3
+numpy==1.26.4
outlier-utils==0.0.5
-pyarrow==14.0.2
+pyarrow==15.0.0
pandas==2.1.4
-scikit-learn==1.3.2
-scipy==1.11.4
+scikit-learn==1.4.1.post1
+scipy==1.12.0
typing_extensions==4.5.0
fancy-collections==0.2.1
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 916bcf3e48bcb984fe6cba28b5270297130bef33..43448cd8ef05f2c8817f28c862750f2417715431 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -12,7 +12,7 @@ import warnings
from copy import copy as shallowcopy
from copy import deepcopy
from functools import partial
-from typing import Any, Hashable, Iterable, MutableMapping, overload
+from typing import Any, Hashable, Iterable, MutableMapping
import numpy as np
import pandas as pd
@@ -32,7 +32,7 @@ from saqc.funcs import FunctionsMixin
# warnings
pd.set_option("mode.chained_assignment", "warn")
-pd.options.mode.copy_on_write = False
+pd.set_option("mode.copy_on_write", True)
np.seterr(invalid="ignore")
diff --git a/saqc/funcs/constants.py b/saqc/funcs/constants.py
index a3af13d614af1f036cf2571a29fdc2cd27692052..3e4afba3a4b125f2dcf68c94a4cd0cdb3bff098b 100644
--- a/saqc/funcs/constants.py
+++ b/saqc/funcs/constants.py
@@ -51,18 +51,19 @@ class ConstantsMixin:
thresh :
Maximum total change allowed per window.
+ window :
+ Size of the moving window. This determines the number of observations used
+ for calculating the absolute change per window.
+ Each window will either contain a fixed number of periods (integer defined window),
+ or will have a fixed temporal extension (offset defined window).
+
min_periods :
Minimum number of observations in window required to generate
- a flag. Must be an integer greater or equal `2`, because a
+ a flag. This can be used to exclude underpopulated *offset* defined windows from
+ flagging. (Integer defined windows will always contain exactly *window* samples).
+ Must be an integer greater or equal `2`, because a
single value would always be considered constant.
Defaults to `2`.
-
- window :
- Size of the moving window. This is the number of observations used
- for calculating the statistic. Each window will be a fixed size.
- If it is an offset then this will be the time period of each window.
- Each window will be a variable sized based on the observations included
- in the time-period.
"""
d: pd.Series = self._data[field]
validateWindow(window, index=d.index)
@@ -73,7 +74,7 @@ class ConstantsMixin:
rolling = d.rolling(window=window, min_periods=min_periods)
starting_points_mask = rolling.max() - rolling.min() <= thresh
- removeRollingRamps(starting_points_mask, window=window, inplace=True)
+ starting_points_mask = removeRollingRamps(starting_points_mask, window=window)
# mimic forward rolling by roll over inverse [::-1]
rolling = starting_points_mask[::-1].rolling(
diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py
index 3167097c3addf74b1dd8492c5458912484a19c8c..029665a3f624beea6614fb2f577d4e8d1046175a 100644
--- a/saqc/funcs/flagtools.py
+++ b/saqc/funcs/flagtools.py
@@ -97,6 +97,64 @@ class FlagtoolsMixin:
self._flags[unflagged, field] = flag
return self
+ @flagging()
+ def setFlags(
+ self,
+ field: str,
+ data: str | list | np.array | pd.Series,
+ override: bool = False,
+ flag: float = BAD,
+ **kwargs,
+ ) -> "SaQC":
+ """
+ Include flags listed in external data.
+
+ Parameters
+ ----------
+
+ data :
+ Determines which timestamps to set flags at, depending on the passed type:
+
+ * 1-d `array` or `List` of timestamps or `pandas.Index`: flag `field` with `flag` at every timestamp in `f_data`
+ * 2-d `array` or List of tuples: for all elements `t[k]` out of f_data:
+ flag `field` with `flag` at every timestamp in between `t[k][0]` and `t[k][1]`
+ * pd.Series: flag `field` with `flag` in between any index and data value of the passed series
+ * str: use the variable timeseries `f_data` as flagging template
+ * pd.Series: flag `field` with `flag` in between any index and data value of the passed series
+ * 1-d `array` or `List` of timestamps: flag `field` with `flag` at every timestamp in `f_data`
+ * 2-d `array` or List of tuples: for all elements `t[k]` out of f_data:
+ flag `field` with `flag` at every timestamp in between `t[k][0]` and `t[k][1]`
+ override :
+ determines if flags shall be assigned although the value in question already has a flag assigned.
+ """
+ to_flag = pd.Series(False, index=self._data[field].index)
+
+ # check if f_data is meant to denote timestamps:
+ if (isinstance(data, (list, np.ndarray, pd.Index))) and not isinstance(
+ data[0], (tuple, np.ndarray)
+ ):
+ set_idx = pd.DatetimeIndex(data).intersection(to_flag.index)
+ to_flag[set_idx] = True
+ else: # f_data denotes intervals:
+ if isinstance(data, (str, pd.Series)):
+ if isinstance(data, str):
+ flags_data = self._data[data]
+ else:
+ flags_data = data
+ intervals = flags_data.items()
+ else:
+ intervals = data
+ for s in intervals:
+ to_flag[s[0] : s[1]] = True
+
+ # elif isinstance(f_data, list):
+ if not override:
+ to_flag &= (self._flags[field] < flag) & (
+ self._flags[field] >= kwargs["dfilter"]
+ )
+ self._flags[to_flag.values, field] = flag
+ return self
+
@register(mask=["field"], demask=["field"], squeeze=["field"])
def flagManual(
self: "SaQC",
@@ -115,6 +173,9 @@ class FlagtoolsMixin:
The method allows to integrate pre-existing flagging information.
+ .. deprecated:: 2.6.0
+ Deprecated Function. See :py:meth:`~saqc.SaQC.setFlags`.
+
Parameters
----------
mdata :
@@ -222,6 +283,11 @@ class FlagtoolsMixin:
2000-05-01 True
dtype: bool
"""
+ warnings.warn(
+ "`flagManual` is deprecated and will be removed in version 2.9 of saqc. "
+ "Please use `setFlags` for similar functionality.",
+ DeprecationWarning,
+ )
validateChoice(
method, "method", ["left-open", "right-open", "closed", "plain", "ontime"]
)
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index 7b5a4b93094c53b856963af5ee3c57e0910b08ae..dfb1a84ef8a51ddba8a2f7a43b31ec60e8549499 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -179,6 +179,8 @@ class OutliersMixin:
p: int = 1,
density: Literal["auto"] | float = "auto",
fill_na: bool = True,
+ slope_correct: bool = True,
+ min_offset: float = None,
flag: float = BAD,
**kwargs,
) -> "SaQC":
@@ -247,6 +249,15 @@ class OutliersMixin:
fill_na :
If True, NaNs in the data are filled with a linear interpolation.
+ slope_correct :
+ if True, a correction is applied, that removes outlier cluster that actually
+ just seem to be steep slopes
+
+ min_offset :
+ If set, only those outlier cluster will be flagged, that are preceeded and succeeeded
+ by sufficiently large value "jumps". Defaults to estimating the sufficient value jumps from
+ the median over the absolute step sizes between data points.
+
See Also
--------
:ref:`introduction to outlier detection with
@@ -366,8 +377,47 @@ class OutliersMixin:
s_mask = ((_s - _s.mean()) / _s.std()).iloc[: int(s.shape[0])].abs() > 3
else:
s_mask = s < -abs(thresh)
-
s_mask = ~isflagged(qc._flags[field], kwargs["dfilter"]) & s_mask
+
+ if slope_correct:
+ g_mask = s_mask.diff()
+ g_mask = g_mask.cumsum()
+ dat = self._data[field]
+ od_groups = dat.interpolate("linear").groupby(by=g_mask)
+ first_vals = od_groups.first()
+ last_vals = od_groups.last()
+ max_vals = od_groups.max()
+ min_vals = od_groups.min()
+ if min_offset is None:
+ if density == "auto":
+ d_diff = dat.diff()
+ eps = d_diff.abs().median()
+ if eps == 0:
+ eps = d_diff[d_diff != 0].abs().median()
+ else:
+ eps = density
+ eps = 3 * eps
+ else:
+ eps = min_offset
+ up_slopes = (min_vals + eps >= last_vals.shift(1)) & (
+ max_vals - eps <= first_vals.shift(-1)
+ )
+ down_slopes = (max_vals - eps <= last_vals.shift(1)) & (
+ min_vals + eps >= first_vals.shift(-1)
+ )
+ slopes = up_slopes | down_slopes
+ odd_return_pred = (max_vals > last_vals.shift(1)) & (
+ min_vals < last_vals.shift(1)
+ )
+ odd_return_succ = (max_vals > first_vals.shift(-1)) & (
+ min_vals < first_vals.shift(-1)
+ )
+ returns = odd_return_succ | odd_return_pred
+ corrections = returns | slopes
+ for s_id in corrections[corrections].index:
+ correct_idx = od_groups.get_group(s_id).index
+ s_mask[correct_idx] = False
+
qc._flags[s_mask, field] = flag
qc = qc.dropField(tmp_field)
return qc
diff --git a/saqc/funcs/scores.py b/saqc/funcs/scores.py
index 18f4c6f62abc8d6426d65c6cd99af3e212b728f8..9998c153477390add9805a11f1eb2c27c21ca09d 100644
--- a/saqc/funcs/scores.py
+++ b/saqc/funcs/scores.py
@@ -503,9 +503,10 @@ class ScoresMixin:
filled = pd.Series(False, index=vals.index)
if density == "auto":
- density = vals.diff().abs().median()
+ v_diff = vals.diff()
+ density = v_diff.abs().median()
if density == 0:
- density = vals.diff().abs().mean()
+ density = v_diff[v_diff != 0].abs().median()
elif isinstance(density, Callable):
density = density(vals)
if isinstance(density, pd.Series):
diff --git a/saqc/funcs/tools.py b/saqc/funcs/tools.py
index 7da335d548321df1dd4d9fa0fb0fab91843c414c..e0f53fabbc32a8b6a89c2d0af1a68267f1dfc565 100644
--- a/saqc/funcs/tools.py
+++ b/saqc/funcs/tools.py
@@ -8,29 +8,154 @@
from __future__ import annotations
import pickle
+import tkinter as tk
import warnings
from typing import TYPE_CHECKING, Optional
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
+import pandas as pd
from typing_extensions import Literal
-from saqc import FILTER_NONE, UNFLAGGED
+from saqc import BAD, FILTER_NONE, UNFLAGGED
from saqc.core import processing, register
from saqc.lib.checking import validateChoice
from saqc.lib.docs import DOC_TEMPLATES
from saqc.lib.plotting import makeFig
-from saqc.lib.tools import periodicMask
+from saqc.lib.selectionGUI import MplScroller, SelectionOverlay
+from saqc.lib.tools import periodicMask, toSequence
if TYPE_CHECKING:
from saqc import SaQC
_MPL_DEFAULT_BACKEND = mpl.get_backend()
+_TEST_MODE = False
class ToolsMixin:
+ @register(mask=[], demask=[], squeeze=[], multivariate=True)
+ def flagByClick(
+ self: "SaQC",
+ field: str | list[str],
+ max_gap: str | None = None,
+ gui_mode: Literal["GUI", "overlay"] = "GUI",
+ selection_marker_kwargs: dict | None = None,
+ dfilter: float = BAD,
+ **kwargs,
+ ) -> "SaQC":
+ """
+ Pop up GUI for adding or removing flags by selection of points in the data plot.
+
+ * Left click and Drag the selection area over the points you want to add to selection.
+
+ * Right clack and drag the selection area over the points you want to remove from selection
+
+ * press 'shift' to switch between rectangle and span selector
+
+ * press 'enter' or click "Assign Flags" to assign flags to the selected points and end session
+
+ * press 'escape' or click "Discard" to end Session without assigneing flags to selection
+
+ * activate the sliders attached to each axes to bind the respective variable. When using the
+ span selector, points from all bound variables will be added synchronously.
+
+
+ Note, that you can only mark already flagged values, if `dfilter` is set accordingly.
+
+ Note, that you can use `flagByClick` to "unflag" already flagged values, when setting `dfilter` above the flag to
+ "unset", and setting `flag` to a flagging level associated with your "unflagged" level.
+
+ Parameters
+ ----------
+ max_gap :
+ If ``None``, all data points will be connected, resulting in long linear
+ lines, in case of large data gaps. ``NaN`` values will be removed before
+ plotting. If an offset string is passed, only points that have a distance
+ below ``max_gap`` are connected via the plotting line.
+ gui_mode :
+ * ``"GUI"`` (default), spawns TK based pop-up GUI, enabling scrolling and binding for subplots
+ * ``"overlay"``, spawns matplotlib based pop-up GUI. May be less conflicting, but does not support
+ scrolling or binding.
+ """
+ data, flags = self._data.copy(), self._flags.copy()
+
+ flag = kwargs.get("flag", BAD)
+ scrollbar = True if gui_mode == "GUI" else False
+ selection_marker_kwargs = selection_marker_kwargs or {}
+
+ if not scrollbar:
+ plt.rcParams["toolbar"] = "toolmanager"
+
+ if not _TEST_MODE:
+ plt.close("all")
+ mpl.use(_MPL_DEFAULT_BACKEND)
+ else:
+ mpl.use("Agg")
+
+ # make base figure, the gui will wrap
+ fig = makeFig(
+ data=data,
+ field=field,
+ flags=flags,
+ level=UNFLAGGED,
+ mode="subplots",
+ max_gap=max_gap,
+ history="valid",
+ xscope=None,
+ ax_kwargs={"ncols": 1},
+ scatter_kwargs={},
+ plot_kwargs={},
+ )
+
+ overlay_data = []
+ for f in field:
+ overlay_data.extend([(data[f][flags[f] < dfilter]).dropna()])
+
+ if scrollbar: # spawn TK based GUI
+ root = tk.Tk()
+ scroller = MplScroller(root, fig=fig)
+ root.protocol("WM_DELETE_WINDOW", scroller.assignAndQuitFunc())
+ scroller.pack(side="top", fill="both", expand=True)
+
+ else: # only use figure window overlay
+ scroller = None
+
+ selector = SelectionOverlay(
+ fig.axes,
+ data=overlay_data,
+ selection_marker_kwargs=selection_marker_kwargs,
+ parent=scroller,
+ )
+ if _TEST_MODE & scrollbar:
+ root.after(2000, root.destroy)
+ # return self
+
+ if scrollbar:
+ root.attributes("-fullscreen", True)
+ root.mainloop()
+ if not _TEST_MODE:
+ root.destroy()
+ else: # show figure if only overlay is used
+ plt.show(block=~_TEST_MODE)
+ plt.rcParams["toolbar"] = "toolbar2"
+
+ # disconnect mouse events when GUI is closed
+ selector.disconnect()
+
+ # assign flags only if selection was confirmed by user
+ if selector.confirmed:
+ for k in range(selector.N):
+ to_flag = selector.index[k][selector.marked[k]]
+
+ new_col = pd.Series(np.nan, index=self._flags[field[k]].index)
+ new_col.loc[to_flag] = flag
+ self._flags.history[field[k]].append(
+ new_col, {"func": "flagByClick", "args": (), "kwargs": kwargs}
+ )
+ return self
+
@register(
mask=[],
demask=[],
@@ -211,7 +336,8 @@ class ToolsMixin:
max_gap: str | None = None,
mode: Literal["subplots", "oneplot"] | str = "oneplot",
history: Literal["valid", "complete"] | list[str] | None = "valid",
- xscope: slice | None = None,
+ xscope: slice | str | None = None,
+ yscope: tuple | list[tuple] | dict = None,
store_kwargs: dict | None = None,
ax: mpl.axes.Axes | None = None,
ax_kwargs: dict | None = None,
@@ -266,6 +392,11 @@ class ToolsMixin:
Determine a chunk of the data to be plotted. ``xscope`` can be anything,
that is a valid argument to the ``pandas.Series.__getitem__`` method.
+ yscope :
+ Either a tuple of 2 scalars that determines all plots' y-view limits, or a list of those
+ tuples, determining the different variables y-view limits (must match number of variables)
+ or a dictionary with variables as keys and the y-view tuple as values.
+
ax :
If not ``None``, plot into the given ``matplotlib.Axes`` instance, instead of a
newly created ``matplotlib.Figure``. This option offers a possibility to integrate
@@ -366,9 +497,20 @@ class ToolsMixin:
marker_kwargs = marker_kwargs or {}
plot_kwargs = plot_kwargs or {}
+ if (
+ (yscope is not None)
+ and (len(yscope) == 2)
+ and not isinstance(yscope[0], (list, tuple))
+ ):
+ yscope = tuple(yscope)
+ if yscope is not None:
+
+ ax_kwargs.update({"ylim": yscope})
+
if not path:
mpl.use(_MPL_DEFAULT_BACKEND)
else:
+ plt.close("all") # supress matplotlib deprecation warning
mpl.use("Agg")
fig = makeFig(
diff --git a/saqc/lib/plotting.py b/saqc/lib/plotting.py
index 72a6cbce11a0680f8b97dee0cace66fbfcb76228..5dd0e944b93d93e1e0b15ddd723bb88d0b4fd5e3 100644
--- a/saqc/lib/plotting.py
+++ b/saqc/lib/plotting.py
@@ -250,7 +250,7 @@ def makeFig(
mode,
)
- # readability formattin fo the x-tick labels:
+ # readability formattin for the x-tick labels:
fig.autofmt_xdate()
return fig
@@ -278,7 +278,7 @@ def _instantiateAxesContext(
next(_scatter_kwargs["marker"])
# assign variable specific labels/titles
- for axis_spec in ["xlabel", "ylabel", "title"]:
+ for axis_spec in ["xlabel", "ylabel", "title", "ylim"]:
spec = _ax_kwargs.get(axis_spec, None)
if isinstance(spec, list):
_ax_kwargs[axis_spec] = spec[var_num]
diff --git a/saqc/lib/selectionGUI.py b/saqc/lib/selectionGUI.py
new file mode 100644
index 0000000000000000000000000000000000000000..6001a3f43ac82b0e64a4a1b96e87e8c8057d95f6
--- /dev/null
+++ b/saqc/lib/selectionGUI.py
@@ -0,0 +1,365 @@
+#! /usr/bin/env python
+
+# SPDX-FileCopyrightText: 2021 Helmholtz-Zentrum für Umweltforschung GmbH - UFZ
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# -*- coding: utf-8 -*-
+
+import tkinter as tk
+
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+from matplotlib.backend_tools import ToolBase
+from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2Tk
+from matplotlib.dates import date2num
+from matplotlib.widgets import RectangleSelector, Slider, SpanSelector
+
+ASSIGN_SHORTCUT = "enter"
+DISCARD_SHORTCUT = "escape"
+LEFT_MOUSE_BUTTON = 1
+RIGHT_MOUSE_BUTTON = 3
+SELECTION_MARKER_DEFAULT = {"zorder": 10, "c": "red", "s": 50, "marker": "x"}
+# if scrollable GUI: determines number of figures per x-size of the screen
+FIGS_PER_SCREEN = 2
+# or hight in inches (if given overrides number of figs per screen):
+FIG_HIGHT_INCH = None
+SLIDER_WIDTH_INCH, SLIDER_HIGHT_INCH = 0.3, 0.2
+BFONT = ("Times", "16")
+VARFONT = ("Times", "12")
+CP_WIDTH = 15
+SELECTOR_DICT = {"rect": "Rectangular", "span": "Span"}
+
+
+class MplScroller(tk.Frame):
+ def __init__(self, parent, fig):
+ tk.Frame.__init__(self, parent)
+ # frame - canvas - window combo that enables scrolling:
+ self.canvas = tk.Canvas(self, borderwidth=0, background="#ffffff")
+ self.canvas.bind_all("<Button-4>", lambda x: self.mouseWheeler(-1))
+ self.canvas.bind_all("<Button-5>", lambda x: self.mouseWheeler(1))
+
+ self.frame = tk.Frame(self.canvas, background="#ffffff")
+ self.vert_scrollbar = tk.Scrollbar(
+ self, orient="vertical", command=self.canvas.yview
+ )
+ self.canvas.configure(yscrollcommand=self.vert_scrollbar.set)
+
+ self.vert_scrollbar.pack(side="right", fill="y")
+ self.canvas.pack(side="left", fill="both", expand=True)
+ self.canvas.create_window(
+ (4, 4), window=self.frame, anchor="nw", tags="self.frame"
+ )
+
+ self.frame.bind("<Configure>", self.scrollAreaCallBack)
+
+ # keeping references
+ self.parent = parent
+ self.fig = fig
+
+ self.control_panel = tk.Frame(self.canvas, bg="DarkGray")
+ self.control_panel.pack(side=tk.LEFT, anchor="n")
+
+ # adding buttons
+ self.quit_button = tk.Button(
+ self.control_panel,
+ text="Discard and Quit.",
+ command=self.assignAndQuitFunc,
+ bg="red",
+ width=CP_WIDTH,
+ relief="flat",
+ overrelief="groove",
+ )
+ self.quit_button.grid(column=3, row=0, pady=5.5, padx=2.25)
+
+ # selector info display
+ self.current_slc_entry = tk.StringVar(self.control_panel)
+
+ tk.Label(
+ self.control_panel,
+ textvariable=self.current_slc_entry,
+ width=CP_WIDTH,
+ # font=BFONT,
+ ).grid(column=1, row=0, pady=5.5, padx=2.25)
+
+ # adjusting content to the scrollable view
+ self.figureSizer()
+ self.figureShifter()
+
+ # add scrollable content
+ self.scrollContentGenerator()
+
+ # add sliders that enable binding variables
+ self.binding_sliders = [None] * len(self.fig.axes)
+ self.binding_status = [False] * len(self.fig.axes)
+ self.makeSlider()
+
+ def mouseWheeler(self, direction):
+ self.canvas.yview_scroll(direction, "units")
+
+ def assignationGenerator(self, selector):
+ tk.Button(
+ self.control_panel,
+ text="Assign Flags",
+ command=lambda s=selector: self.assignAndQuitFunc(s),
+ bg="green",
+ width=CP_WIDTH,
+ ).grid(column=0, row=0, pady=5.5, padx=2.25)
+
+ def assignAndQuitFunc(self, selector=None):
+ if selector:
+ selector.confirmed = True
+ plt.close(self.fig)
+ self.quit()
+
+ def scrollContentGenerator(self):
+ canvas = FigureCanvasTkAgg(self.fig, master=self.frame)
+ toolbar = NavigationToolbar2Tk(canvas, self.canvas)
+ toolbar.update()
+ toolbar.pack(side=tk.TOP)
+ canvas.get_tk_widget().pack()
+ canvas.draw()
+
+ def scrollAreaCallBack(self, event):
+ self.canvas.configure(scrollregion=self.canvas.bbox("all"))
+
+ def figureSizer(self):
+ manager = plt.get_current_fig_manager()
+ if hasattr(manager, "window"):
+ window = plt.get_current_fig_manager().window
+ f_size = list(window.wm_maxsize())
+ else: # for testing mode
+ f_size = [1, 1]
+ px = 1 / plt.rcParams["figure.dpi"]
+ f_size = [ws * px for ws in f_size]
+ if not FIG_HIGHT_INCH:
+ f_size[1] = f_size[1] * len(self.fig.axes) * FIGS_PER_SCREEN**-1
+ else:
+ f_size[1] = FIG_HIGHT_INCH * len(self.fig.axes)
+ self.fig.set_size_inches(f_size[0], f_size[1])
+
+ def figureShifter(self):
+ manager = plt.get_current_fig_manager()
+ if hasattr(manager, "window"):
+ window = manager.window
+ screen_hight = window.wm_maxsize()[1]
+ else: # for testing mode
+ screen_hight = 10
+ fig_hight = self.fig.get_size_inches()
+ ratio = fig_hight[1] / screen_hight
+ to_shift = ratio
+ for k in range(len(self.fig.axes)):
+ b = self.fig.axes[k].get_position().bounds
+ self.fig.axes[k].set_position((b[0], b[1] + to_shift, b[2], b[3]))
+
+ def makeSlider(self):
+ fig_sz = self.fig.get_size_inches()
+ slider_width, slider_hight = (
+ SLIDER_WIDTH_INCH / fig_sz[0],
+ SLIDER_HIGHT_INCH / fig_sz[1],
+ )
+ for ax in enumerate(self.fig.axes):
+ b0 = ax[1].get_position().get_points()
+ b0_ax = plt.axes([b0[0, 0], b0[1, 1], slider_width, slider_hight])
+ self.binding_sliders[ax[0]] = Slider(b0_ax, "", 0, 1, valinit=0, valstep=1)
+ self.binding_sliders[ax[0]].valtext.set_visible(False)
+ self.binding_sliders[ax[0]].on_changed(
+ lambda val, ax_num=ax[0]: self.bindFunc(val, ax_num)
+ )
+
+ def bindFunc(self, val, ax_num):
+ self.binding_status[ax_num] = bool(val)
+
+
+class AssignFlagsTool(ToolBase):
+ description = "Assign flags to selection."
+
+ def __init__(self, *args, callback, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.callback = callback
+
+ def trigger(self, *args, **kwargs):
+ self.callback()
+
+
+class SelectionOverlay:
+ def __init__(
+ self, ax, data, selection_marker_kwargs=SELECTION_MARKER_DEFAULT, parent=None
+ ):
+ self.parent = parent
+ self.N = len(data)
+ self.ax = ax
+ self.marker_handles = self.N * [None]
+ for k in range(self.N):
+ self.ax[k].set_xlim(auto=True)
+
+ self.canvas = self.ax[0].figure.canvas
+ self.selection_marker_kwargs = {
+ **SELECTION_MARKER_DEFAULT,
+ **selection_marker_kwargs,
+ }
+ self.rc_rect = None
+ self.lc_rect = None
+ self.current_slc = "rect"
+ self.spawn_selector(type=self.current_slc)
+ self.marked = [np.zeros(data[k].shape[0]).astype(bool) for k in range(self.N)]
+ self.confirmed = False
+ self.index = [data[k].index for k in range(self.N)]
+ self.data = [data[k].values for k in range(self.N)]
+ self.numidx = [date2num(self.index[k]) for k in range(self.N)]
+ if (not parent) and (not (matplotlib.get_backend() == "agg")):
+ # add assignment button to the toolbar
+ self.canvas.manager.toolmanager.add_tool(
+ "Assign Flags", AssignFlagsTool, callback=self.assignAndCloseCB
+ )
+ self.canvas.manager.toolbar.add_tool("Assign Flags", "Flags")
+ self.canvas.manager.toolmanager.remove_tool("help")
+ elif parent:
+ parent.assignationGenerator(self)
+
+ self.canvas.mpl_connect("key_press_event", self.keyPressEvents)
+ self.canvas.draw_idle()
+
+ def onLeftSelectFunc(self, ax_num):
+ return lambda x, y, z=ax_num: self.onLeftSelect(x, y, z)
+
+ def onRightSelectFunc(self, ax_num):
+ return lambda x, y, z=ax_num: self.onRightSelect(x, y, z)
+
+ def onLeftSelect(self, eclick, erelease, ax_num=0, _select_to=True):
+ ax_num = np.array([ax_num])
+ s_mask = {}
+ if (self.current_slc == "span") and (self.parent is not None):
+ stati = np.array(self.parent.binding_status)
+ if stati.any():
+ stati_w = np.where(stati)[0]
+ if ax_num[0] in stati_w:
+ ax_num = stati_w
+
+ if self.current_slc == "rect":
+ upper_left = (
+ min(eclick.xdata, erelease.xdata),
+ max(eclick.ydata, erelease.ydata),
+ )
+
+ bottom_right = (
+ max(eclick.xdata, erelease.xdata),
+ min(eclick.ydata, erelease.ydata),
+ )
+ x_cut = (self.numidx[ax_num[0]] > upper_left[0]) & (
+ self.numidx[ax_num[0]] < bottom_right[0]
+ )
+ y_cut = (self.data[ax_num[0]] > bottom_right[1]) & (
+ self.data[ax_num[0]] < upper_left[1]
+ )
+ s_mask.update({ax_num[0]: x_cut & y_cut})
+
+ if self.current_slc == "span":
+ for a in ax_num:
+ x_cut = (self.numidx[a] > eclick) & (self.numidx[a] < erelease)
+ s_mask.update({a: x_cut})
+
+ for num in ax_num:
+ self.marked[num][s_mask[num]] = _select_to
+ xl = self.ax[num].get_xlim()
+ yl = self.ax[num].get_ylim()
+ marker_artist = self.ax[num].scatter(
+ self.index[num][self.marked[num]],
+ self.data[num][self.marked[num]],
+ **self.selection_marker_kwargs,
+ )
+
+ if self.marker_handles[num] is not None:
+ self.marker_handles[num].remove()
+ self.marker_handles[num] = marker_artist
+ self.ax[num].set_xlim(xl)
+ self.ax[num].set_ylim(yl)
+
+ self.canvas.draw_idle()
+
+ def onRightSelect(self, eclick, erelease, ax_num=0):
+ self.onLeftSelect(eclick, erelease, ax_num=ax_num, _select_to=False)
+
+ def disconnect(self):
+ for k in range(self.N):
+ self.lc_rect[k].disconnect_events()
+ self.rc_rect[k].disconnect_events()
+
+ def spawn_selector(self, type="rect"):
+ if self.rc_rect:
+ for k in range(self.N):
+ self.rc_rect[k].disconnect_events()
+ self.lc_rect[k].disconnect_events()
+ if type == "rect":
+ self.lc_rect = [
+ RectangleSelector(
+ self.ax[k],
+ self.onLeftSelectFunc(k),
+ button=[1],
+ use_data_coordinates=True,
+ useblit=True,
+ )
+ for k in range(self.N)
+ ]
+ self.rc_rect = [
+ RectangleSelector(
+ self.ax[k],
+ self.onRightSelectFunc(k),
+ button=[3],
+ use_data_coordinates=True,
+ useblit=True,
+ )
+ for k in range(self.N)
+ ]
+ elif type == "span":
+ self.lc_rect = [
+ SpanSelector(
+ self.ax[k],
+ self.onLeftSelectFunc(k),
+ "horizontal",
+ button=[1],
+ useblit=True,
+ )
+ for k in range(self.N)
+ ]
+ self.rc_rect = [
+ SpanSelector(
+ self.ax[k],
+ self.onRightSelectFunc(k),
+ "horizontal",
+ button=[3],
+ useblit=True,
+ )
+ for k in range(self.N)
+ ]
+ if self.parent:
+ self.parent.current_slc_entry.set(SELECTOR_DICT[self.current_slc])
+
+ def assignAndCloseCB(self, val=None):
+ self.confirmed = True
+ plt.close(self.ax[0].figure)
+
+ def discardAndCloseCB(self, val=None):
+ plt.close(self.ax[0].figure)
+
+ def keyPressEvents(self, event):
+ if event.key == ASSIGN_SHORTCUT:
+ if self.parent is None:
+ self.assignAndCloseCB()
+ else:
+ self.parent.assignAndQuitFunc(self)
+ if event.key == DISCARD_SHORTCUT:
+ if self.parent is None:
+ self.discardAndCloseCB()
+ else:
+ self.parent.assignAndQuitFunc(None)
+
+ elif event.key == "shift":
+ if self.current_slc == "rect":
+ self.current_slc = "span"
+ self.spawn_selector("span")
+
+ elif self.current_slc == "span":
+ self.current_slc = "rect"
+ self.spawn_selector("rect")
diff --git a/tests/core/test_flags.py b/tests/core/test_flags.py
index cc8949bf620cdabb4da6aa5ef72f9552a35d379c..c9628d49cd7f333e9fc7527dd05db48ddaf0cffc 100644
--- a/tests/core/test_flags.py
+++ b/tests/core/test_flags.py
@@ -122,7 +122,7 @@ def test_copy(data: Union[pd.DataFrame, DictOfSeries, Dict[str, pd.Series]]):
# the underling series data is the same
for c in shallow.columns:
- assert shallow._data[c].index is flags._data[c].index
+ assert shallow._data[c].index.equals(flags._data[c].index)
# the underling series data was copied
for c in deep.columns:
diff --git a/tests/core/test_history.py b/tests/core/test_history.py
index e8279f46237482d0d90c7d29bc00480c664015e1..cb3412c370f90046bf14a062176932ed0bee9984 100644
--- a/tests/core/test_history.py
+++ b/tests/core/test_history.py
@@ -143,7 +143,7 @@ def test_copy(data):
assert is_equal(deep, shallow)
# underling pandas data was only copied with deep=True
- assert shallow.hist.index is hist.hist.index
+ assert shallow.hist.index.equals(hist.hist.index)
assert deep.hist.index is not hist.hist.index
diff --git a/tests/funcs/test_flagtools.py b/tests/funcs/test_flagtools.py
index eb4081bdf5750b20b29412d9d67a5900c298be0f..6bda00301e2bd493dadff6fa2c2205b46467419b 100644
--- a/tests/funcs/test_flagtools.py
+++ b/tests/funcs/test_flagtools.py
@@ -11,6 +11,7 @@ import numpy as np
import pandas as pd
import pytest
+import saqc
from saqc import BAD as B
from saqc import UNFLAGGED as U
from saqc import SaQC
@@ -218,3 +219,82 @@ def test_transferFlags():
assert qc3._history["a"].hist.iloc[:, 0].equals(qc3._history["x"].hist.squeeze())
assert qc3._history["a"].hist.iloc[:, 1].equals(qc3._history["y"].hist.squeeze())
assert qc3._history["a"].hist.iloc[:, 2].equals(qc3._history["z"].hist.squeeze())
+
+
+@pytest.mark.parametrize(
+ "f_data",
+ [
+ (
+ pd.Series(
+ ["2000-01-01T00:30:00", "2000-01-01T01:30:00"],
+ index=["2000-01-01T00:00:00", "2000-01-01T01:00:00"],
+ )
+ ),
+ (
+ np.array(
+ [
+ ("2000-01-01T00:00:00", "2000-01-01T00:30:00"),
+ ("2000-01-01T01:00:00", "2000-01-01T01:30:00"),
+ ]
+ )
+ ),
+ (
+ [
+ ("2000-01-01T00:00:00", "2000-01-01T00:30:00"),
+ ("2000-01-01T01:00:00", "2000-01-01T01:30:00"),
+ ]
+ ),
+ ("maint"),
+ ],
+)
+def test_setFlags_intervals(f_data):
+ start = ["2000-01-01T00:00:00", "2000-01-01T01:00:00"]
+ end = ["2000-01-01T00:30:00", "2000-01-01T01:30:00"]
+ maint_data = pd.Series(data=end, index=pd.DatetimeIndex(start), name="maint")
+ data = pd.Series(
+ np.arange(30),
+ index=pd.date_range("2000", freq="11min", periods=30),
+ name="data",
+ )
+ qc = saqc.SaQC([data, maint_data])
+ qc = qc.setFlags("data", data=f_data)
+ assert (qc.flags["data"].iloc[np.r_[0:3, 6:9]] > 0).all()
+ assert (qc.flags["data"].iloc[np.r_[4:6, 10:30]] < 0).all()
+
+
+@pytest.mark.parametrize(
+ "f_data",
+ [
+ (
+ np.array(
+ [
+ "2000-01-01T00:00:00",
+ "2000-01-01T00:30:00",
+ "2000-01-01T01:00:00",
+ "2000-01-01T01:30:00",
+ ]
+ )
+ ),
+ (
+ [
+ "2000-01-01T00:00:00",
+ "2000-01-01T00:30:00",
+ "2000-01-01T01:00:00",
+ "2000-01-01T01:30:00",
+ ]
+ ),
+ ],
+)
+def test_setFlags_ontime(f_data):
+ start = ["2000-01-01T00:00:00", "2000-01-01T01:00:00"]
+ end = ["2000-01-01T00:30:00", "2000-01-01T01:30:00"]
+ maint_data = pd.Series(data=end, index=pd.DatetimeIndex(start), name="maint")
+ data = pd.Series(
+ np.arange(30),
+ index=pd.date_range("2000", freq="11min", periods=30),
+ name="data",
+ )
+ qc = saqc.SaQC([data, maint_data])
+ qc = qc.setFlags("data", data=f_data)
+ assert qc.flags["data"].iloc[0] > 0
+ assert (qc.flags["data"].iloc[1:] < 0).all()
diff --git a/tests/funcs/test_functions.py b/tests/funcs/test_functions.py
index 3065d9f66c05a87e2af73fa3553d12477f9358b5..8812569ae3b584983e3e41f1d5266994458341ab 100644
--- a/tests/funcs/test_functions.py
+++ b/tests/funcs/test_functions.py
@@ -179,19 +179,21 @@ def test_flagManual(data, field):
]
for kw in kwargs_list:
- qc = SaQC(data, flags).flagManual(field, **kw)
+ with pytest.deprecated_call():
+ qc = SaQC(data, flags).flagManual(field, **kw)
isflagged = qc._flags[field] > UNFLAGGED
assert isflagged[isflagged].index.equals(index_exp)
# flag not exist in mdata
- qc = SaQC(data, flags).flagManual(
- field,
- mdata=mdata,
- mflag="i do not exist",
- method="ontime",
- mformat="mflag",
- flag=BAD,
- )
+ with pytest.deprecated_call():
+ qc = SaQC(data, flags).flagManual(
+ field,
+ mdata=mdata,
+ mflag="i do not exist",
+ method="ontime",
+ mformat="mflag",
+ flag=BAD,
+ )
isflagged = qc._flags[field] > UNFLAGGED
assert isflagged[isflagged].index.equals(pd.DatetimeIndex([]))
@@ -220,14 +222,15 @@ def test_flagManual(data, field):
]
bound_drops = {"right-open": [1], "left-open": [0], "closed": []}
for method in ["right-open", "left-open", "closed"]:
- qc = qc.flagManual(
- field,
- mdata=mdata,
- mflag=1,
- method=method,
- mformat="mflag",
- flag=BAD,
- )
+ with pytest.deprecated_call():
+ qc = qc.flagManual(
+ field,
+ mdata=mdata,
+ mflag=1,
+ method=method,
+ mformat="mflag",
+ flag=BAD,
+ )
isflagged = qc._flags[field] > UNFLAGGED
for flag_i in flag_intervals:
f_i = isflagged[slice(flag_i[0], flag_i[-1])].index
diff --git a/tests/funcs/test_tools.py b/tests/funcs/test_tools.py
index 028bae5d8480872b7a1b070ec92c72b711753e99..231ab5faef59e1b9aa1111bd8bce4803acc3430d 100644
--- a/tests/funcs/test_tools.py
+++ b/tests/funcs/test_tools.py
@@ -32,10 +32,27 @@ def test_makeFig(tmp_path):
outfile = str(Path(tmp_path, "test.png")) # the filesystem's temp dir
d_saqc = d_saqc.plot(field="data", path=outfile)
- d_saqc = d_saqc.plot(field="data", path=outfile, history="valid", stats=True)
+ d_saqc = d_saqc.plot(
+ field="data", path=outfile, history="valid", yscope=[(-50, 1000)]
+ )
with pytest.deprecated_call():
d_saqc = d_saqc.plot(field="data", path=outfile, history="complete")
d_saqc = d_saqc.plot(
- field="data", path=outfile, ax_kwargs={"ylabel": "data is data"}, stats=True
+ field="data",
+ path=outfile,
+ ax_kwargs={"ylabel": "data is data"},
+ yscope=(100, 150),
+ )
+
+
+@pytest.mark.filterwarnings("ignore::UserWarning")
+def test_flagByClick():
+ saqc.funcs.tools._TEST_MODE = True
+ data = pd.DataFrame(
+ {f"d{k}": np.random.randint(0, 100, 100) for k in range(10)},
+ index=pd.date_range("2000", freq="1d", periods=100),
)
+ qc = saqc.SaQC(data)
+ qc = qc.flagByClick(data.columns, gui_mode="overlay")
+ qc = qc.flagByClick(data.columns)
diff --git a/tests/requirements.txt b/tests/requirements.txt
index fe7b3b98b3d9e1f2533100e87395f32bf45d6933..fe4007473f8b81486001de7b2b4c7cf4fe31ea98 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -2,9 +2,9 @@
#
# SPDX-License-Identifier: GPL-3.0-or-later
-beautifulsoup4==4.12.2
-hypothesis==6.92.2
-Markdown==3.5.1
+beautifulsoup4==4.12.3
+hypothesis==6.98.15
+Markdown==3.5.2
pytest==7.4.4
pytest-lazy-fixture==0.6.3
requests==2.31.0